Tell independent fabrics apart by their subnet manager, not their name
A fabric was what agent.yaml called it. With several independent InfiniBand fabrics at a site, two given one name would have had a replica spread across both, starting and then hanging on its first collective. Each active InfiniBand port now reports the master subnet manager it sees (sminfo, asked once a minute), which every port on one fabric shares and ports on separate fabrics cannot; a RoCE port reports the IP network of its RoCE v2 address. Ports also carry their node and port GUIDs. Placement divides a declared fabric by that physical identity and forms a replica on one part only; a node whose adapters for one name are on two fabrics is refused, and a node that cannot say is kept apart from those that can. /api/v1/fabrics and the cluster page list each physical fabric with its subnet manager, and say when one name covers two fabrics, two names one fabric, an adapter is up but undeclared, or a node cannot be checked. doctor reports each port's subnet manager, and an agent on a node with adapters installs infiniband-diags when sminfo is missing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
51f1606b53
commit
77fe801892
21 files changed
+1130
-58
No files matched your search
@@ -2,6 +2,7 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -24,6 +25,9 @@ type DashboardData struct {
|
||||
Unplaced []adminapi.ExplainView
|
||||
Usage []store.KeyUsage
|
||||
Summary adminapi.Summary
|
||||
// Fabrics is the physical fabrics and what disagrees about them, from a
|
||||
// source that can say.
|
||||
Fabrics *fabrics.Report
|
||||
|
||||
Users []store.UserUsage
|
||||
Models2 []store.ModelUsage
|
||||
@@ -91,6 +95,12 @@ type Source interface {
|
||||
Summary() adminapi.Summary
|
||||
}
|
||||
|
||||
// FabricSource is a Source that can say which physical fabrics the adapters
|
||||
// are on. Optional, so a source without one still draws the page.
|
||||
type FabricSource interface {
|
||||
Fabrics() fabrics.Report
|
||||
}
|
||||
|
||||
// UsageSource is the accounting half, absent when authentication is off.
|
||||
type UsageSource interface {
|
||||
UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
|
||||
@@ -165,6 +175,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
}
|
||||
d.Unplaced = unplaced(src, d.Models)
|
||||
d.Summary = src.Summary()
|
||||
if fs, ok := src.(FabricSource); ok {
|
||||
r := fs.Fabrics()
|
||||
d.Fabrics = &r
|
||||
}
|
||||
}
|
||||
if usage != nil {
|
||||
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
|
||||
|
||||
@@ -548,3 +548,6 @@ table.held tr.model-group > td { background: var(--bg); padding-top: .7rem; bord
|
||||
table.held tr.model-group strong { font-size: .95rem; }
|
||||
table.held tr.model-version > td:first-child { padding-left: 1.4rem; }
|
||||
.tag.bad { color: var(--bad); border-color: var(--bad); }
|
||||
ul.fabric-problems { list-style: none; padding: 0; margin: .25rem 0 .75rem; font-size: .88rem; }
|
||||
ul.fabric-problems li { margin: .25rem 0; }
|
||||
ul.fabric-problems li.info { color: var(--muted); }
|
||||
@@ -130,18 +130,26 @@
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Summary.Fabrics}}
|
||||
{{if or .Summary.Fabrics .Fabrics}}
|
||||
<h2>Fabrics</h2>
|
||||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}<tr>
|
||||
{{with .Fabrics}}{{if .Problems}}
|
||||
<ul class="fabric-problems">{{range .Problems}}
|
||||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}
|
||||
</ul>{{end}}{{end}}
|
||||
{{if .Summary.Fabrics}}
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}{{$name := .Fabric}}<tr>
|
||||
<td><code>{{.Fabric}}</code></td>
|
||||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||||
<td class="num">{{.Nodes}}</td>
|
||||
<td class="num">{{.LargestFreeBlock}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||||
for a model that needs this one.</p>
|
||||
{{end}}
|
||||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||||
unused nodes, which is what decides whether such a replica can land.</p>
|
||||
{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
|
||||
@@ -35,12 +35,13 @@
|
||||
<h2>Fabric</h2>
|
||||
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
|
||||
{{if .Node.IBPorts}}
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
{{range .Node.IBPorts}}<tr>
|
||||
<td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
|
||||
<td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
|
||||
<td class="num">{{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}}</td>
|
||||
<td>{{or .Fabric "—"}}</td>
|
||||
<td>{{if .Physical}}{{.Physical}}{{else if .SMError}}<span class="note" title="{{.SMError}}">unknown</span>{{else}}—{{end}}</td>
|
||||
<td>{{if .Errors}}{{range $k, $v := .Errors}}<span class="tag bad">{{$k}} {{$v}}</span> {{end}}{{else}}<span class="note">none</span>{{end}}</td>
|
||||
<td class="num">{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}</td>
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"html/template"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -103,6 +104,7 @@ var funcs = template.FuncMap{
|
||||
// about the size of the same file starts an argument about neither.
|
||||
"bytes": humanBytes,
|
||||
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
|
||||
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
|
||||
// dict lets one chart template be called with named arguments. Go
|
||||
// templates pass a single value, and four charts differing in three
|
||||
// strings each is four copies of the same markup otherwise.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -649,3 +650,23 @@ func TestTheClusterPageLeadsWithHealthAndRefreshesInPlace(t *testing.T) {
|
||||
t.Error("without script the page should still refresh")
|
||||
}
|
||||
}
|
||||
|
||||
// fabricSource is fakeSource that can say what its fabrics physically are.
|
||||
type fabricSource struct{ fakeSource }
|
||||
|
||||
func (fabricSource) Fabrics() fabrics.Report {
|
||||
return fabrics.Report{
|
||||
Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)",
|
||||
Declared: []string{"ib-a"}, RateGbps: 400}},
|
||||
Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) {
|
||||
_, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user