Tell independent fabrics apart by their subnet manager, not their name
A fabric was what agent.yaml called it. With several independent InfiniBand fabrics at a site, two given one name would have had a replica spread across both, starting and then hanging on its first collective. Each active InfiniBand port now reports the master subnet manager it sees (sminfo, asked once a minute), which every port on one fabric shares and ports on separate fabrics cannot; a RoCE port reports the IP network of its RoCE v2 address. Ports also carry their node and port GUIDs. Placement divides a declared fabric by that physical identity and forms a replica on one part only; a node whose adapters for one name are on two fabrics is refused, and a node that cannot say is kept apart from those that can. /api/v1/fabrics and the cluster page list each physical fabric with its subnet manager, and say when one name covers two fabrics, two names one fabric, an adapter is up but undeclared, or a node cannot be checked. doctor reports each port's subnet manager, and an agent on a node with adapters installs infiniband-diags when sminfo is missing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
51f1606b53
commit
77fe801892
21 files changed
+1130
-58
No files matched your search
@@ -1398,7 +1398,25 @@ type IBPort struct {
|
||||
// link_layer is InfiniBand or Ethernet (RoCE).
|
||||
LinkLayer string `protobuf:"bytes,6,opt,name=link_layer,json=linkLayer,proto3" json:"link_layer,omitempty"`
|
||||
// counters are cumulative, by sysfs name; data counters in bytes.
|
||||
Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
|
||||
Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
|
||||
// Identity: the adapter's node GUID and the port's own, as the fabric
|
||||
// knows them.
|
||||
NodeGuid string `protobuf:"bytes,8,opt,name=node_guid,json=nodeGuid,proto3" json:"node_guid,omitempty"`
|
||||
PortGuid string `protobuf:"bytes,9,opt,name=port_guid,json=portGuid,proto3" json:"port_guid,omitempty"`
|
||||
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
||||
// one physical fabric sees the same one, which is what tells separate
|
||||
// fabrics apart when their names are only what somebody typed.
|
||||
SmGuid string `protobuf:"bytes,10,opt,name=sm_guid,json=smGuid,proto3" json:"sm_guid,omitempty"`
|
||||
SmLid int32 `protobuf:"varint,11,opt,name=sm_lid,json=smLid,proto3" json:"sm_lid,omitempty"`
|
||||
SmPriority int32 `protobuf:"varint,12,opt,name=sm_priority,json=smPriority,proto3" json:"sm_priority,omitempty"`
|
||||
SmState string `protobuf:"bytes,13,opt,name=sm_state,json=smState,proto3" json:"sm_state,omitempty"`
|
||||
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
||||
// infiniband-diags missing, or no subnet manager answering.
|
||||
SmError string `protobuf:"bytes,14,opt,name=sm_error,json=smError,proto3" json:"sm_error,omitempty"`
|
||||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"`
|
||||
GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1482,6 +1500,69 @@ func (x *IBPort) GetCounters() map[string]uint64 {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *IBPort) GetNodeGuid() string {
|
||||
if x != nil {
|
||||
return x.NodeGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetPortGuid() string {
|
||||
if x != nil {
|
||||
return x.PortGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmGuid() string {
|
||||
if x != nil {
|
||||
return x.SmGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmLid() int32 {
|
||||
if x != nil {
|
||||
return x.SmLid
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmPriority() int32 {
|
||||
if x != nil {
|
||||
return x.SmPriority
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmState() string {
|
||||
if x != nil {
|
||||
return x.SmState
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmError() string {
|
||||
if x != nil {
|
||||
return x.SmError
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetNetwork() string {
|
||||
if x != nil {
|
||||
return x.Network
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetGidPrefix() string {
|
||||
if x != nil {
|
||||
return x.GidPrefix
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
@@ -2787,7 +2868,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x98\x02\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x92\x04\n" +
|
||||
"\x06IBPort\x12\x10\n" +
|
||||
"\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" +
|
||||
"\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" +
|
||||
@@ -2797,7 +2878,19 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\trate_gbps\x18\x05 \x01(\x05R\brateGbps\x12\x1d\n" +
|
||||
"\n" +
|
||||
"link_layer\x18\x06 \x01(\tR\tlinkLayer\x12:\n" +
|
||||
"\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x1a;\n" +
|
||||
"\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x12\x1b\n" +
|
||||
"\tnode_guid\x18\b \x01(\tR\bnodeGuid\x12\x1b\n" +
|
||||
"\tport_guid\x18\t \x01(\tR\bportGuid\x12\x17\n" +
|
||||
"\asm_guid\x18\n" +
|
||||
" \x01(\tR\x06smGuid\x12\x15\n" +
|
||||
"\x06sm_lid\x18\v \x01(\x05R\x05smLid\x12\x1f\n" +
|
||||
"\vsm_priority\x18\f \x01(\x05R\n" +
|
||||
"smPriority\x12\x19\n" +
|
||||
"\bsm_state\x18\r \x01(\tR\asmState\x12\x19\n" +
|
||||
"\bsm_error\x18\x0e \x01(\tR\asmError\x12\x18\n" +
|
||||
"\anetwork\x18\x0f \x01(\tR\anetwork\x12\x1d\n" +
|
||||
"\n" +
|
||||
"gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x1a;\n" +
|
||||
"\rCountersEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" +
|
||||
|
||||
@@ -277,6 +277,25 @@ message IBPort {
|
||||
string link_layer = 6;
|
||||
// counters are cumulative, by sysfs name; data counters in bytes.
|
||||
map<string, uint64> counters = 7;
|
||||
|
||||
// Identity: the adapter's node GUID and the port's own, as the fabric
|
||||
// knows them.
|
||||
string node_guid = 8;
|
||||
string port_guid = 9;
|
||||
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
||||
// one physical fabric sees the same one, which is what tells separate
|
||||
// fabrics apart when their names are only what somebody typed.
|
||||
string sm_guid = 10;
|
||||
int32 sm_lid = 11;
|
||||
int32 sm_priority = 12;
|
||||
string sm_state = 13;
|
||||
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
||||
// infiniband-diags missing, or no subnet manager answering.
|
||||
string sm_error = 14;
|
||||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
string network = 15;
|
||||
string gid_prefix = 16;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
|
||||
+87
-2
@@ -16,6 +16,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -357,6 +358,12 @@ type Agent struct {
|
||||
lastWeights string
|
||||
// fabricsSent is what the control plane was last told about fabrics.
|
||||
fabricsSent string
|
||||
// sm caches which subnet manager each InfiniBand port sees, asked once
|
||||
// a minute: a management datagram per port per heartbeat is more than
|
||||
// an answer that changes on a failover is worth.
|
||||
sm map[string]*discovery.SMInfo
|
||||
smErr map[string]string
|
||||
smAsked time.Time
|
||||
// load measures the machine between heartbeats.
|
||||
load *discovery.LoadSampler
|
||||
|
||||
@@ -431,6 +438,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
a.log.Warn("cannot create the weights directory", "dir", a.cfg.WeightsCache, "err", err)
|
||||
}
|
||||
go a.provisionLoop(ctx)
|
||||
go a.ensureDiags(ctx)
|
||||
|
||||
attempt := 0
|
||||
next := 0
|
||||
@@ -647,12 +655,22 @@ func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
a.attachSM(hcas)
|
||||
for _, h := range hcas {
|
||||
for _, p := range h.Ports {
|
||||
hb.IbPorts = append(hb.IbPorts, &llmgwv1.IBPort{
|
||||
ip := &llmgwv1.IBPort{
|
||||
Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
|
||||
RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
|
||||
})
|
||||
NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
|
||||
}
|
||||
if p.SM != nil {
|
||||
ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
|
||||
} else {
|
||||
a.mu.Lock()
|
||||
ip.SmError = a.smErr[smKey(h.Name, p.Num)]
|
||||
a.mu.Unlock()
|
||||
}
|
||||
hb.IbPorts = append(hb.IbPorts, ip)
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
@@ -1608,3 +1626,70 @@ func ResolveManagement(c Config) (addr, iface string) {
|
||||
}
|
||||
return addr, iface
|
||||
}
|
||||
|
||||
func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
|
||||
|
||||
// querySM is discovery.QuerySM, replaced in tests.
|
||||
var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
|
||||
return discovery.QuerySM(context.Background(), hca, port)
|
||||
}
|
||||
|
||||
// attachSM puts on each active InfiniBand port the subnet manager it sees,
|
||||
// asking again once a minute and otherwise reusing the last answer.
|
||||
func (a *Agent) attachSM(hcas []discovery.HCA) {
|
||||
a.mu.Lock()
|
||||
stale := time.Since(a.smAsked) > time.Minute || a.sm == nil
|
||||
a.mu.Unlock()
|
||||
if stale {
|
||||
sm, errs := map[string]*discovery.SMInfo{}, map[string]string{}
|
||||
_ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) {
|
||||
got, err := querySM(hca, port)
|
||||
if err != nil {
|
||||
errs[smKey(hca, port)] = err.Error()
|
||||
return nil, err
|
||||
}
|
||||
sm[smKey(hca, port)] = got
|
||||
return got, nil
|
||||
})
|
||||
a.mu.Lock()
|
||||
for k, v := range sm {
|
||||
if old := a.sm[k]; old != nil && old.GUID != v.GUID {
|
||||
a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID)
|
||||
}
|
||||
}
|
||||
a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for i := range hcas {
|
||||
for j := range hcas[i].Ports {
|
||||
if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() {
|
||||
hcas[i].Ports[j].SM = sm
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureDiags installs infiniband-diags on a node with adapters and without
|
||||
// it. sminfo is what tells which physical fabric a port is on; without it,
|
||||
// two separate fabrics given one name cannot be caught. Installed the way
|
||||
// engine dependencies are, from the machine's own package sources.
|
||||
func (a *Agent) ensureDiags(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
hasIB := len(a.inv.HCAs) > 0
|
||||
a.mu.Unlock()
|
||||
if !hasIB || a.prov == nil {
|
||||
return
|
||||
}
|
||||
if _, err := exec.LookPath("sminfo"); err == nil {
|
||||
return
|
||||
}
|
||||
a.log.Info("installing infiniband-diags, to tell which fabric each port is on")
|
||||
pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}}
|
||||
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
||||
a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager",
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package discovery
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -52,6 +53,40 @@ type Port struct {
|
||||
// default of 0 is a RoCE v1 link-local entry on most setups, which does
|
||||
// not route, and a job on it hangs.
|
||||
RoCEv2GID int `json:"rocev2_gid"`
|
||||
// PortGUID is the port's own identifier, the interface half of its GID.
|
||||
PortGUID string `json:"port_guid,omitempty"`
|
||||
// SM is the subnet manager this port sees as master, from sminfo. Nil on
|
||||
// RoCE, which has none, and when infiniband-diags is not installed.
|
||||
SM *SMInfo `json:"sm,omitempty"`
|
||||
// Network is, on RoCE, the IP network of the RoCE v2 address: the only
|
||||
// thing that says two RoCE ports are on the same fabric.
|
||||
Network string `json:"network,omitempty"`
|
||||
}
|
||||
|
||||
// SMInfo is a subnet manager as a port sees it.
|
||||
type SMInfo struct {
|
||||
GUID string `json:"guid"`
|
||||
LID int `json:"lid"`
|
||||
Priority int `json:"priority"`
|
||||
// State is SMINFO_MASTER, SMINFO_STANDBY, ...
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// FabricKey is what identifies the physical fabric this port is on: the
|
||||
// master subnet manager on InfiniBand, since every port on one fabric sees
|
||||
// the same one and ports on separate fabrics cannot; the IP network on RoCE.
|
||||
// Empty when neither is known, and for a port that is not up.
|
||||
func (p Port) FabricKey() string {
|
||||
if !p.Active() {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case p.LinkLayer == LinkEthernet && p.Network != "":
|
||||
return "roce:" + p.Network
|
||||
case p.SM != nil && p.SM.GUID != "":
|
||||
return "sm:" + p.SM.GUID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// PortCounters are the counters worth watching, from each port's counters/
|
||||
@@ -80,6 +115,9 @@ type HCA struct {
|
||||
// FirmwareVersion is reported by the driver; worth capturing because
|
||||
// NCCL behaviour differs between firmware revisions.
|
||||
FirmwareVersion string `json:"firmware_version,omitempty"`
|
||||
// NodeGUID identifies the adapter on its fabric: it is the name a
|
||||
// switch's view of the fabric, and every other tool, knows it by.
|
||||
NodeGUID string `json:"node_guid,omitempty"`
|
||||
}
|
||||
|
||||
// ActivePorts returns the ports that can carry traffic.
|
||||
@@ -126,6 +164,7 @@ func DiscoverIB(root string) ([]HCA, error) {
|
||||
func readHCA(dir, name string) (HCA, error) {
|
||||
h := HCA{Name: name, NUMANode: -1}
|
||||
h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver"))
|
||||
h.NodeGUID = normalizeGUID(readTrimmed(filepath.Join(dir, "node_guid")))
|
||||
|
||||
// device is a symlink into the PCI tree; its target's base name is the
|
||||
// bus address, and numa_node lives beside it.
|
||||
@@ -163,8 +202,14 @@ func readPort(dir string, num int) Port {
|
||||
SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
|
||||
}
|
||||
p.RoCEv2GID = -1
|
||||
p.PortGUID = portGUID(readTrimmed(filepath.Join(dir, "gids", "0")))
|
||||
if p.LinkLayer == LinkEthernet {
|
||||
p.RoCEv2GID = rocev2GID(dir)
|
||||
if p.RoCEv2GID >= 0 {
|
||||
idx := strconv.Itoa(p.RoCEv2GID)
|
||||
p.Network = RoCENetwork(readTrimmed(filepath.Join(dir, "gid_attrs", "ndevs", idx)),
|
||||
readTrimmed(filepath.Join(dir, "gids", idx)))
|
||||
}
|
||||
}
|
||||
for _, name := range PortCounters {
|
||||
raw := readTrimmed(filepath.Join(dir, "counters", name))
|
||||
@@ -262,3 +307,72 @@ func rocev2GID(dir string) int {
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// normalizeGUID writes a GUID one way: sysfs says "0c42:a103:0045:bd6c",
|
||||
// sminfo and ibnetdiscover say "0x0c42a1030045bd6c".
|
||||
func normalizeGUID(g string) string {
|
||||
g = strings.ToLower(strings.TrimSpace(g))
|
||||
g = strings.TrimPrefix(g, "0x")
|
||||
g = strings.ReplaceAll(g, ":", "")
|
||||
if g == "" || strings.Trim(g, "0") == "" {
|
||||
return ""
|
||||
}
|
||||
for len(g) < 16 {
|
||||
g = "0" + g
|
||||
}
|
||||
return "0x" + g
|
||||
}
|
||||
|
||||
// NormalizeGUID is normalizeGUID for other packages.
|
||||
func NormalizeGUID(g string) string { return normalizeGUID(g) }
|
||||
|
||||
// portGUID is the interface identifier of a GID: its last four groups.
|
||||
func portGUID(gid string) string {
|
||||
parts := strings.Split(gid, ":")
|
||||
if len(parts) != 8 {
|
||||
return ""
|
||||
}
|
||||
return normalizeGUID(strings.Join(parts[4:], ""))
|
||||
}
|
||||
|
||||
// RoCENetwork is the IP network an IPv4-mapped RoCE GID's address sits in,
|
||||
// read from the interface that carries it. Replaced in tests.
|
||||
var RoCENetwork = func(ndev, gid string) string {
|
||||
ip := gidIPv4(gid)
|
||||
if ip == nil || ndev == "" {
|
||||
return ""
|
||||
}
|
||||
ifc, err := net.InterfaceByName(ndev)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
addrs, _ := ifc.Addrs()
|
||||
for _, a := range addrs {
|
||||
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
|
||||
return (&net.IPNet{IP: ip.Mask(n.Mask), Mask: n.Mask}).String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// gidIPv4 is the IPv4 address in an IPv4-mapped GID, nil otherwise.
|
||||
func gidIPv4(gid string) net.IP {
|
||||
g := strings.ToLower(gid)
|
||||
const mapped = "0000:0000:0000:0000:0000:ffff:"
|
||||
if !strings.HasPrefix(g, mapped) {
|
||||
return nil
|
||||
}
|
||||
hex := strings.ReplaceAll(strings.TrimPrefix(g, mapped), ":", "")
|
||||
if len(hex) != 8 {
|
||||
return nil
|
||||
}
|
||||
b := make(net.IP, 4)
|
||||
for i := range 4 {
|
||||
v, err := strconv.ParseUint(hex[i*2:i*2+2], 16, 8)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
b[i] = byte(v)
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -55,6 +55,7 @@ func fakeSysfs(t *testing.T) string {
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
|
||||
write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/ndevs/3", "ens1f0")
|
||||
|
||||
// A PCI device with a NUMA node, linked the way the driver does it.
|
||||
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
||||
@@ -260,3 +261,65 @@ func TestRoCEFindsItsRoutableGID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSubnetManagerAPortSeesIsRead(t *testing.T) {
|
||||
sm, err := ParseSMInfo("sminfo: sm lid 1 sm guid 0x98039B03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sm.GUID != "0x98039b03009fcfd6" || sm.LID != 1 || sm.Priority != 15 || sm.State != "SMINFO_MASTER" {
|
||||
t.Errorf("sm = %+v", sm)
|
||||
}
|
||||
if _, err := ParseSMInfo("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port"); err == nil {
|
||||
t.Error("an error from sminfo should not parse")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPortsFabricIsItsSubnetManagerOrItsNetwork(t *testing.T) {
|
||||
// Two ports on one InfiniBand fabric see one master SM, and on separate
|
||||
// fabrics cannot; RoCE has no SM, and is its IP network.
|
||||
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hcas[0].NodeGUID != "" {
|
||||
t.Errorf("no node_guid file, yet %q", hcas[0].NodeGUID)
|
||||
}
|
||||
err = AttachSM(hcas, func(hca string, port int) (*SMInfo, error) {
|
||||
return &SMInfo{GUID: "0x00000000000000aa", LID: 1, State: "SMINFO_MASTER"}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := hcas[0].Ports[0].FabricKey(); got != "sm:0x00000000000000aa" {
|
||||
t.Errorf("IB key = %q", got)
|
||||
}
|
||||
if got := hcas[1].Ports[1].FabricKey(); got != "" {
|
||||
t.Errorf("a port that is down is on no fabric, got %q", got)
|
||||
}
|
||||
if hcas[2].Ports[0].SM != nil {
|
||||
t.Error("RoCE was asked for a subnet manager")
|
||||
}
|
||||
if got := hcas[0].Ports[0].PortGUID; got != "0x0002c90300a01234" {
|
||||
t.Errorf("port GUID = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARoCEPortsNetworkIsItsFabric(t *testing.T) {
|
||||
old := RoCENetwork
|
||||
t.Cleanup(func() { RoCENetwork = old })
|
||||
RoCENetwork = func(ndev, gid string) string {
|
||||
if ip := gidIPv4(gid); ip != nil && ip.String() == "10.0.0.1" {
|
||||
return "10.0.0.0/24"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
roce := hcas[2].Ports[0]
|
||||
if roce.Network != "10.0.0.0/24" || roce.FabricKey() != "roce:10.0.0.0/24" {
|
||||
t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
|
||||
// cannot be asked who it is.
|
||||
var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")
|
||||
|
||||
// smLine is sminfo's one line of answer:
|
||||
//
|
||||
// sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
|
||||
var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)
|
||||
|
||||
// ParseSMInfo reads sminfo's output.
|
||||
func ParseSMInfo(out string) (*SMInfo, error) {
|
||||
m := smLine.FindStringSubmatch(out)
|
||||
if m == nil {
|
||||
return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
|
||||
}
|
||||
lid, _ := strconv.Atoi(m[1])
|
||||
prio, _ := strconv.Atoi(m[3])
|
||||
return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
|
||||
}
|
||||
|
||||
// QuerySM asks, through one port, which subnet manager is master. It is a
|
||||
// management datagram to the SM, not a scan: cheap, but not free, which is
|
||||
// why the agent asks once a minute rather than every heartbeat.
|
||||
func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
|
||||
bin, err := exec.LookPath("sminfo")
|
||||
if err != nil {
|
||||
return nil, ErrNoDiags
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return ParseSMInfo(string(out))
|
||||
}
|
||||
|
||||
// AttachSM fills in the subnet manager each active InfiniBand port sees.
|
||||
// RoCE has none. The first error is returned for reporting; the ports it
|
||||
// could answer for keep their answer.
|
||||
func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
|
||||
var first error
|
||||
for i := range hcas {
|
||||
for j := range hcas[i].Ports {
|
||||
p := &hcas[i].Ports[j]
|
||||
if !p.Active() || p.LinkLayer == LinkEthernet {
|
||||
continue
|
||||
}
|
||||
sm, err := query(hcas[i].Name, p.Num)
|
||||
if err != nil {
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
p.SM = sm
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -48,6 +49,9 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
|
||||
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
|
||||
mux.HandleFunc("GET /api/v1/nodes", a.listNodes)
|
||||
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Fabrics())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/models", a.listModels)
|
||||
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
|
||||
// Listing is a read and deleting is a separate verb, so a mistyped
|
||||
@@ -134,6 +138,10 @@ type IBPortView struct {
|
||||
Active bool `json:"active"`
|
||||
// Fabric is the declared fabric this port's adapter carries, if any.
|
||||
Fabric string `json:"fabric,omitempty"`
|
||||
// Physical is the fabric the port is physically on: its subnet manager,
|
||||
// or its RoCE network. SMError says why that is not known.
|
||||
Physical string `json:"physical,omitempty"`
|
||||
SMError string `json:"sm_error,omitempty"`
|
||||
// Errors are the error counters that are not zero, by name.
|
||||
Errors map[string]uint64 `json:"errors,omitempty"`
|
||||
TxBytes uint64 `json:"tx_bytes,omitempty"`
|
||||
@@ -336,7 +344,13 @@ func (a *API) Nodes() []NodeView {
|
||||
for _, p := range n.IBPorts {
|
||||
pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState,
|
||||
RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA],
|
||||
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]}
|
||||
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"], SMError: p.SMError}
|
||||
switch {
|
||||
case p.RoCE() && p.Network != "":
|
||||
pv.Physical = "RoCE " + p.Network
|
||||
case p.SMGUID != "":
|
||||
pv.Physical = fmt.Sprintf("SM %s (LID %d)", p.SMGUID, p.SMLID)
|
||||
}
|
||||
for name, c := range p.Counters {
|
||||
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
|
||||
if pv.Errors == nil {
|
||||
@@ -818,3 +832,9 @@ func (a *API) profileChanged(model, profile, was string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Fabrics is the physical fabrics the adapters are on, against the names
|
||||
// the nodes gave them.
|
||||
func (a *API) Fabrics() fabrics.Report {
|
||||
return fabrics.Analyze(a.ctl.Inventory().Nodes())
|
||||
}
|
||||
@@ -411,6 +411,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
|
||||
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
|
||||
NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(),
|
||||
SMGUID: p.GetSmGuid(), SMLID: int(p.GetSmLid()), SMPriority: int(p.GetSmPriority()),
|
||||
SMState: p.GetSmState(), SMError: p.GetSmError(), Network: p.GetNetwork(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// Package fabrics works out which physical fabrics the cluster's adapters
|
||||
// are on, and where that disagrees with what the nodes were told.
|
||||
//
|
||||
// A fabric's name is configuration. What it is physically is what the
|
||||
// adapters report: on InfiniBand the master subnet manager every port on one
|
||||
// fabric sees, on RoCE the IP network. Kept apart so a site with several
|
||||
// independent fabrics can see each, and see at once when two were given one
|
||||
// name, one was given two, or an adapter is cabled somewhere nobody declared.
|
||||
package fabrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Level is how much a problem matters.
|
||||
type Level string
|
||||
|
||||
// The levels.
|
||||
const (
|
||||
Fail Level = "fail"
|
||||
Warn Level = "warn"
|
||||
Info Level = "info"
|
||||
)
|
||||
|
||||
// Problem is one disagreement, in words.
|
||||
type Problem struct {
|
||||
Level Level `json:"level"`
|
||||
Fabric string `json:"fabric,omitempty"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Physical is one fabric as the adapters find it.
|
||||
type Physical struct {
|
||||
Key string `json:"key"`
|
||||
// Kind is infiniband or roce.
|
||||
Kind string `json:"kind"`
|
||||
// Label is the key as a person reads it.
|
||||
Label string `json:"label"`
|
||||
// SM is the master subnet manager, on InfiniBand.
|
||||
SM *SM `json:"sm,omitempty"`
|
||||
// Declared are the names nodes gave this fabric; one, when all is well.
|
||||
Declared []string `json:"declared"`
|
||||
Members []Member `json:"members"`
|
||||
RateGbps int `json:"rate_gbps"`
|
||||
GIDPrefix string `json:"gid_prefix,omitempty"`
|
||||
}
|
||||
|
||||
// SM is a subnet manager as the ports see it.
|
||||
type SM struct {
|
||||
GUID string `json:"guid"`
|
||||
LID int `json:"lid"`
|
||||
Priority int `json:"priority"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// Member is one node's adapters on a physical fabric.
|
||||
type Member struct {
|
||||
Node string `json:"node"`
|
||||
HCAs []string `json:"hcas"`
|
||||
// Declared is the name this node gives these adapters, empty when it
|
||||
// declares them in no fabric.
|
||||
Declared string `json:"declared,omitempty"`
|
||||
}
|
||||
|
||||
// Report is the whole picture.
|
||||
type Report struct {
|
||||
Physical []Physical `json:"physical"`
|
||||
Problems []Problem `json:"problems"`
|
||||
// Unverified are declared fabrics some of whose nodes cannot say which
|
||||
// physical fabric they are on, with why.
|
||||
Unverified map[string][]string `json:"unverified,omitempty"`
|
||||
}
|
||||
|
||||
// Analyze reads every node's ports.
|
||||
func Analyze(nodes []inventory.Node) Report {
|
||||
r := Report{Unverified: map[string][]string{}}
|
||||
byKey := map[string]*Physical{}
|
||||
declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes
|
||||
|
||||
for _, n := range nodes {
|
||||
declOf := map[string]string{} // hca -> declared fabric
|
||||
for _, f := range n.Fabrics {
|
||||
for _, h := range f.HCAs {
|
||||
declOf[h] = f.Fabric
|
||||
}
|
||||
}
|
||||
// Which physical fabric each declared fabric's adapters are on here.
|
||||
seenDecl := map[string]map[string]bool{}
|
||||
members := map[string]*Member{} // key -> this node's membership
|
||||
|
||||
for _, p := range n.IBPorts {
|
||||
key := p.FabricKey()
|
||||
decl := declOf[p.HCA]
|
||||
if key == "" {
|
||||
if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" {
|
||||
r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError)
|
||||
}
|
||||
continue
|
||||
}
|
||||
ph := byKey[key]
|
||||
if ph == nil {
|
||||
ph = &Physical{Key: key, Kind: "infiniband", GIDPrefix: p.GIDPrefix}
|
||||
if p.RoCE() {
|
||||
ph.Kind, ph.Label = "roce", "RoCE "+p.Network
|
||||
} else {
|
||||
ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState}
|
||||
ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID)
|
||||
}
|
||||
byKey[key] = ph
|
||||
}
|
||||
if p.RateGbps > 0 && (ph.RateGbps == 0 || p.RateGbps < ph.RateGbps) {
|
||||
ph.RateGbps = p.RateGbps
|
||||
}
|
||||
m := members[key]
|
||||
if m == nil {
|
||||
m = &Member{Node: n.ID, Declared: decl}
|
||||
members[key] = m
|
||||
}
|
||||
m.HCAs = appendOnce(m.HCAs, p.HCA)
|
||||
if decl != "" {
|
||||
if seenDecl[decl] == nil {
|
||||
seenDecl[decl] = map[string]bool{}
|
||||
}
|
||||
seenDecl[decl][key] = true
|
||||
ph.Declared = appendOnce(ph.Declared, decl)
|
||||
if declaredKeys[decl] == nil {
|
||||
declaredKeys[decl] = map[string][]string{}
|
||||
}
|
||||
declaredKeys[decl][key] = appendOnce(declaredKeys[decl][key], n.ID)
|
||||
} else {
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Message: fmt.Sprintf(
|
||||
"%s: %s is up on %s but declared in no fabric, so llmgw sends nothing over it",
|
||||
n.ID, p.HCA, ph.Label)})
|
||||
}
|
||||
}
|
||||
for decl, keys := range seenDecl {
|
||||
if len(keys) > 1 {
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
|
||||
"%s: the adapters it declares as %s are cabled to %d different fabrics; it is left out of multi-node work on %s",
|
||||
n.ID, decl, len(keys), decl)})
|
||||
}
|
||||
}
|
||||
for key, m := range members {
|
||||
byKey[key].Members = append(byKey[key].Members, *m)
|
||||
}
|
||||
}
|
||||
|
||||
for decl, keys := range declaredKeys {
|
||||
if len(keys) < 2 {
|
||||
continue
|
||||
}
|
||||
var parts []string
|
||||
for _, k := range sortedKeys(keys) {
|
||||
parts = append(parts, fmt.Sprintf("%s: %s", byKey[k].Label, strings.Join(keys[k], ", ")))
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
|
||||
"%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name",
|
||||
decl, len(keys), strings.Join(parts, "; "))})
|
||||
}
|
||||
for _, ph := range byKey {
|
||||
sort.Strings(ph.Declared)
|
||||
sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node })
|
||||
if len(ph.Declared) > 1 {
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: ph.Declared[0], Message: fmt.Sprintf(
|
||||
"%s are one fabric (%s): a replica could use all of them, but each name is placed on alone",
|
||||
strings.Join(ph.Declared, " and "), ph.Label)})
|
||||
}
|
||||
r.Physical = append(r.Physical, *ph)
|
||||
}
|
||||
sort.Slice(r.Physical, func(i, j int) bool {
|
||||
a, b := r.Physical[i], r.Physical[j]
|
||||
if na, nb := first(a.Declared), first(b.Declared); na != nb {
|
||||
return na < nb
|
||||
}
|
||||
return a.Key < b.Key
|
||||
})
|
||||
|
||||
// Separate InfiniBand fabrics on one subnet prefix: harmless while they
|
||||
// stay apart, and the reason their GIDs cannot tell them apart.
|
||||
byPrefix := map[string][]string{}
|
||||
for _, ph := range r.Physical {
|
||||
if ph.Kind == "infiniband" && ph.GIDPrefix != "" {
|
||||
byPrefix[ph.GIDPrefix] = append(byPrefix[ph.GIDPrefix], ph.Label)
|
||||
}
|
||||
}
|
||||
for prefix, labels := range byPrefix {
|
||||
if len(labels) > 1 {
|
||||
r.Problems = append(r.Problems, Problem{Level: Info, Message: fmt.Sprintf(
|
||||
"%d separate fabrics share the subnet prefix %s; that is the default, and fine while they stay apart — a distinct subnet_prefix per subnet manager makes their GIDs unique",
|
||||
len(labels), prefix)})
|
||||
}
|
||||
}
|
||||
for decl, why := range r.Unverified {
|
||||
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf(
|
||||
"%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))})
|
||||
}
|
||||
|
||||
sort.SliceStable(r.Problems, func(i, j int) bool {
|
||||
if li, lj := rank(r.Problems[i].Level), rank(r.Problems[j].Level); li != lj {
|
||||
return li < lj
|
||||
}
|
||||
return r.Problems[i].Message < r.Problems[j].Message
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
func rank(l Level) int {
|
||||
switch l {
|
||||
case Fail:
|
||||
return 0
|
||||
case Warn:
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
func first(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "~" // undeclared last
|
||||
}
|
||||
return s[0]
|
||||
}
|
||||
|
||||
func appendOnce(s []string, v string) []string {
|
||||
if slices.Contains(s, v) {
|
||||
return s
|
||||
}
|
||||
return append(s, v)
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package fabrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func ibNode(id, declared, sm string, hcas ...string) inventory.Node {
|
||||
n := inventory.Node{ID: id}
|
||||
if declared != "" {
|
||||
n.Fabrics = []inventory.FabricAttachment{{Fabric: declared, HCAs: hcas}}
|
||||
}
|
||||
for _, h := range hcas {
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400,
|
||||
LinkLayer: "InfiniBand", SMGUID: sm, SMLID: 1, GIDPrefix: "fe80:0000:0000:0000"})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func problem(t *testing.T, r Report, l Level, fragment string) {
|
||||
t.Helper()
|
||||
for _, p := range r.Problems {
|
||||
if p.Level == l && strings.Contains(p.Message, fragment) {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Errorf("no %s problem containing %q in %+v", l, fragment, r.Problems)
|
||||
}
|
||||
|
||||
func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"),
|
||||
ibNode("b1", "ib-b", "0x02", "mlx5_0"),
|
||||
})
|
||||
if len(r.Physical) != 2 {
|
||||
t.Fatalf("physical = %+v", r.Physical)
|
||||
}
|
||||
if r.Physical[0].Declared[0] != "ib-a" || len(r.Physical[0].Members) != 2 || r.Physical[0].SM.GUID != "0x01" {
|
||||
t.Errorf("ib-a = %+v", r.Physical[0])
|
||||
}
|
||||
for _, p := range r.Problems {
|
||||
if p.Level != Info {
|
||||
t.Errorf("unexpected problem: %+v", p)
|
||||
}
|
||||
}
|
||||
// Both on the default prefix: said, quietly.
|
||||
problem(t, r, Info, "share the subnet prefix")
|
||||
}
|
||||
|
||||
func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"),
|
||||
})
|
||||
problem(t, r, Fail, "ib-a is 2 separate fabrics")
|
||||
}
|
||||
|
||||
func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"),
|
||||
})
|
||||
problem(t, r, Warn, "ib-a and ib-b are one fabric")
|
||||
}
|
||||
|
||||
func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) {
|
||||
n := ibNode("a1", "ib-a", "0x01", "mlx5_0")
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"})
|
||||
problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09")
|
||||
}
|
||||
|
||||
func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) {
|
||||
n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1")
|
||||
n.IBPorts[1].SMGUID = "0x02"
|
||||
problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
|
||||
}
|
||||
|
||||
func TestANodeThatCannotSayIsReported(t *testing.T) {
|
||||
n := ibNode("x", "ib-a", "", "mlx5_0")
|
||||
n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags"
|
||||
problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags")
|
||||
}
|
||||
|
||||
func TestRoCEIsItsNetwork(t *testing.T) {
|
||||
n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}},
|
||||
IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}}
|
||||
r := Analyze([]inventory.Node{n})
|
||||
if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" {
|
||||
t.Errorf("physical = %+v", r.Physical)
|
||||
}
|
||||
}
|
||||
@@ -120,7 +120,7 @@ type Node struct {
|
||||
IBPorts []IBPort `json:"-"`
|
||||
}
|
||||
|
||||
// IBPort is one port's state and cumulative counters.
|
||||
// IBPort is one port's state, identity and cumulative counters.
|
||||
type IBPort struct {
|
||||
HCA string
|
||||
Port int
|
||||
@@ -129,11 +129,64 @@ type IBPort struct {
|
||||
RateGbps int
|
||||
LinkLayer string
|
||||
Counters map[string]uint64
|
||||
|
||||
NodeGUID string
|
||||
PortGUID string
|
||||
GIDPrefix string
|
||||
// SMGUID and the rest are the master subnet manager this port sees;
|
||||
// SMError says why they are empty on an active InfiniBand port.
|
||||
SMGUID string
|
||||
SMLID int
|
||||
SMPriority int
|
||||
SMState string
|
||||
SMError string
|
||||
// Network is a RoCE port's IP network.
|
||||
Network string
|
||||
}
|
||||
|
||||
// Active reports whether the port can carry traffic.
|
||||
func (p IBPort) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
|
||||
|
||||
// RoCE reports whether the port is Ethernet carrying RDMA.
|
||||
func (p IBPort) RoCE() bool { return strings.EqualFold(p.LinkLayer, "Ethernet") }
|
||||
|
||||
// FabricKey names the physical fabric the port is on: its master subnet
|
||||
// manager on InfiniBand, its IP network on RoCE. Empty when not known, and
|
||||
// for a port that is not up.
|
||||
func (p IBPort) FabricKey() string {
|
||||
if !p.Active() {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case p.RoCE() && p.Network != "":
|
||||
return "roce:" + p.Network
|
||||
case !p.RoCE() && p.SMGUID != "":
|
||||
return "sm:" + p.SMGUID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// PhysicalFabric is the physical fabric this node's adapters for a declared
|
||||
// fabric are on. Known is false when none of them says; ok is false when
|
||||
// they disagree — two adapters declared as one fabric that are cabled to two.
|
||||
func (n Node) PhysicalFabric(fabric string) (key string, known, ok bool) {
|
||||
hcas := n.HCAsFor(fabric)
|
||||
for _, p := range n.IBPorts {
|
||||
if !slices.Contains(hcas, p.HCA) {
|
||||
continue
|
||||
}
|
||||
k := p.FabricKey()
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if key != "" && k != key {
|
||||
return "", true, false
|
||||
}
|
||||
key = k
|
||||
}
|
||||
return key, key != "", true
|
||||
}
|
||||
|
||||
// Load is a node's use of its CPU and memory, and each engine's share.
|
||||
type Load struct {
|
||||
At time.Time
|
||||
|
||||
@@ -247,52 +247,113 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
}
|
||||
|
||||
for _, fabric := range fabrics {
|
||||
pool := fabric + "/" + p.classLabel()
|
||||
candidates := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
|
||||
if len(candidates) < p.Nodes {
|
||||
rejections = append(rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool,
|
||||
Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
// Prefer the emptiest nodes for multi-node work: a replica spanning
|
||||
// nodes is the least movable thing in the cluster, so give it the
|
||||
// cleanest machines rather than squeezing it into the gaps.
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return len(candidates[i].free) > len(candidates[j].free)
|
||||
})
|
||||
|
||||
assignments := make([]NodeAssignment, 0, p.Nodes)
|
||||
for rank, c := range candidates[:p.Nodes] {
|
||||
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric))
|
||||
if gpus == nil {
|
||||
assignments = nil
|
||||
rejections = append(rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool, Node: c.node.ID,
|
||||
Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode),
|
||||
})
|
||||
break
|
||||
all := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
for _, group := range physicalGroups(fabric, all, p, &rejections) {
|
||||
pool := fabric + "/" + p.classLabel()
|
||||
if group.key != "" && group.split {
|
||||
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
|
||||
}
|
||||
if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
}
|
||||
assignments = append(assignments, NodeAssignment{
|
||||
NodeID: c.node.ID,
|
||||
Rank: rank,
|
||||
GPUIndices: gpus,
|
||||
// Only this fabric's adapters, never the node's others.
|
||||
HCAs: c.node.HCAsFor(fabric),
|
||||
})
|
||||
}
|
||||
if assignments == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}, rejections
|
||||
}
|
||||
return nil, rejections
|
||||
}
|
||||
|
||||
// fabricGroup is the candidates on one physical fabric.
|
||||
type fabricGroup struct {
|
||||
key string
|
||||
nodes []candidate
|
||||
// split is set when the declared fabric turned out to be several.
|
||||
split bool
|
||||
}
|
||||
|
||||
// physicalGroups divides a declared fabric's candidates by the physical
|
||||
// fabric their adapters are on.
|
||||
//
|
||||
// A fabric's name is what an operator typed, and two separate fabrics given
|
||||
// one name would have had a replica spread across both: it starts, and hangs
|
||||
// on its first collective, since nothing routes between them. Grouped by the
|
||||
// subnet manager each node's ports see (by IP network on RoCE), a replica
|
||||
// stays on one. Nodes that do not say — an agent without infiniband-diags —
|
||||
// are a group of their own rather than trusted to join one that does.
|
||||
func physicalGroups(fabric string, cands []candidate, p Profile, rejections *[]Rejection) []fabricGroup {
|
||||
by := map[string][]candidate{}
|
||||
for _, c := range cands {
|
||||
key, _, ok := c.node.PhysicalFabric(fabric)
|
||||
if !ok {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: fabric + "/" + p.classLabel(), Node: c.node.ID,
|
||||
Reason: "its adapters declared as " + fabric + " are cabled to different fabrics",
|
||||
})
|
||||
continue
|
||||
}
|
||||
by[key] = append(by[key], c)
|
||||
}
|
||||
out := make([]fabricGroup, 0, len(by))
|
||||
for k, v := range by {
|
||||
out = append(out, fabricGroup{key: k, nodes: v, split: len(by) > 1})
|
||||
}
|
||||
// Largest first, then by key, so the choice is the same every time.
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if len(out[i].nodes) != len(out[j].nodes) {
|
||||
return len(out[i].nodes) > len(out[j].nodes)
|
||||
}
|
||||
return out[i].key < out[j].key
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// shortKey is a physical fabric's key as a pool name carries it.
|
||||
func shortKey(k string) string {
|
||||
if i := strings.IndexByte(k, ':'); i >= 0 {
|
||||
k = k[i+1:]
|
||||
}
|
||||
if len(k) > 10 && strings.HasPrefix(k, "0x") {
|
||||
return "sm-" + k[len(k)-6:]
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// placeOnFabric forms a replica from candidates that share one fabric.
|
||||
func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []candidate, rejections *[]Rejection) *Placement {
|
||||
if len(candidates) < p.Nodes {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool,
|
||||
Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prefer the emptiest nodes for multi-node work: a replica spanning
|
||||
// nodes is the least movable thing in the cluster, so give it the
|
||||
// cleanest machines rather than squeezing it into the gaps.
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return len(candidates[i].free) > len(candidates[j].free)
|
||||
})
|
||||
|
||||
assignments := make([]NodeAssignment, 0, p.Nodes)
|
||||
for rank, c := range candidates[:p.Nodes] {
|
||||
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric))
|
||||
if gpus == nil {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool, Node: c.node.ID,
|
||||
Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
assignments = append(assignments, NodeAssignment{
|
||||
NodeID: c.node.ID,
|
||||
Rank: rank,
|
||||
GPUIndices: gpus,
|
||||
// Only this fabric's adapters, never the node's others.
|
||||
HCAs: c.node.HCAsFor(fabric),
|
||||
})
|
||||
}
|
||||
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}
|
||||
}
|
||||
|
||||
// candidate is a node that passed the hard constraints.
|
||||
type candidate struct {
|
||||
node inventory.Node
|
||||
|
||||
@@ -888,3 +888,73 @@ func TestWithoutTopologyItIsTheLowestIndices(t *testing.T) {
|
||||
t.Errorf("chose %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// onSM puts a node's declared adapters on the fabric a subnet manager runs.
|
||||
func onSM(n inventory.Node, sm string) inventory.Node {
|
||||
for _, f := range n.Fabrics {
|
||||
for _, h := range f.HCAs {
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE",
|
||||
LinkLayer: "InfiniBand", SMGUID: sm})
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestTwoFabricsGivenOneNameAreNeverSpanned(t *testing.T) {
|
||||
// Somebody named two separate fabrics ib-a. Each has two free nodes; the
|
||||
// replica needs three. Spread across both it would start and hang on its
|
||||
// first collective, since nothing routes between them.
|
||||
st := store(
|
||||
onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01"), onSM(node("a2", "h100-80g", 8, "ib-a"), "0x01"),
|
||||
onSM(node("b1", "h100-80g", 8, "ib-a"), "0x02"), onSM(node("b2", "h100-80g", 8, "ib-a"), "0x02"),
|
||||
)
|
||||
res := New(st).Place([]Profile{{Name: "big", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 3}}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatalf("placed across two physical fabrics: %+v", res.Placement.Nodes)
|
||||
}
|
||||
if !strings.Contains(res.Explain(), "ib-a@") {
|
||||
t.Errorf("the explanation should name the physical fabrics:\n%s", res.Explain())
|
||||
}
|
||||
|
||||
// Two nodes fit on either; they come from one.
|
||||
res = New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
first := res.Placement.Nodes[0].NodeID[0]
|
||||
for _, a := range res.Placement.Nodes {
|
||||
if a.NodeID[0] != first {
|
||||
t.Errorf("a pair spans fabrics: %+v", res.Placement.Nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatDoesNotSayIsNotTrustedToJoin(t *testing.T) {
|
||||
// Without infiniband-diags a node cannot say which fabric it is on. It is
|
||||
// kept apart from nodes that can, rather than assumed to be with them.
|
||||
st := store(
|
||||
onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01"),
|
||||
node("u1", "h100-80g", 8, "ib-a"),
|
||||
)
|
||||
if res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil); res.Placement != nil {
|
||||
t.Errorf("a node of unknown fabric was paired with one of known: %+v", res.Placement.Nodes)
|
||||
}
|
||||
// Where no node says, the declared name is all there is, as before.
|
||||
st = store(node("u1", "h100-80g", 8, "ib-a"), node("u2", "h100-80g", 8, "ib-a"))
|
||||
if res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil); res.Placement == nil {
|
||||
t.Errorf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdaptersOfOneDeclaredFabricOnTwoAreRefused(t *testing.T) {
|
||||
n := node("x", "h100-80g", 8, "ib-a")
|
||||
n.IBPorts = []inventory.IBPort{
|
||||
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x01"},
|
||||
{HCA: "mlx5_1", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x02"},
|
||||
}
|
||||
res := New(store(n, onSM(node("y", "h100-80g", 8, "ib-a"), "0x01"))).Place(
|
||||
[]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
|
||||
if res.Placement != nil || !strings.Contains(res.Explain(), "cabled to different fabrics") {
|
||||
t.Errorf("expected a refusal naming the cabling:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
@@ -59,8 +59,10 @@ type Facts struct {
|
||||
NVSwitch bool `json:"nvswitch"`
|
||||
FabricManager string `json:"fabric_manager,omitempty"`
|
||||
|
||||
HCAs []discovery.HCA `json:"hcas"`
|
||||
Modules map[string]bool `json:"modules"`
|
||||
HCAs []discovery.HCA `json:"hcas"`
|
||||
// SMErrors say, per port, why the subnet manager could not be asked.
|
||||
SMErrors map[string]string `json:"sm_errors,omitempty"`
|
||||
Modules map[string]bool `json:"modules"`
|
||||
|
||||
// ConfigPath is the agent configuration read, empty when there was none.
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
@@ -126,6 +128,14 @@ func Collect(ctx context.Context, o Options) Facts {
|
||||
root = discovery.SysfsRoot
|
||||
}
|
||||
f.HCAs, _ = discovery.DiscoverIB(root)
|
||||
f.SMErrors = map[string]string{}
|
||||
_ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) {
|
||||
sm, err := discovery.QuerySM(ctx, hca, port)
|
||||
if err != nil {
|
||||
f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error()
|
||||
}
|
||||
return sm, err
|
||||
})
|
||||
for _, m := range watchedModules {
|
||||
_, err := os.Stat(filepath.Join("/sys/module", m))
|
||||
f.Modules[m] = err == nil
|
||||
@@ -295,7 +305,23 @@ func Evaluate(f Facts) []Finding {
|
||||
add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang",
|
||||
"give the interface an IPv4 address, or check the adapter's RoCE mode")
|
||||
case p.Active():
|
||||
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)", where, p.State, p.RateGbps, p.LinkLayer), "")
|
||||
on := ""
|
||||
switch {
|
||||
case p.SM != nil:
|
||||
on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State)
|
||||
case p.Network != "":
|
||||
on = ", network " + p.Network
|
||||
}
|
||||
add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "")
|
||||
if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil {
|
||||
why := f.SMErrors[where]
|
||||
fix := ""
|
||||
if strings.Contains(why, "infiniband-diags") {
|
||||
fix = "install infiniband-diags"
|
||||
}
|
||||
add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+
|
||||
"; without it, separate fabrics given one name cannot be caught", fix)
|
||||
}
|
||||
if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 {
|
||||
add(Warn, "fabric", where+" is active but reports no subnet manager LID", "")
|
||||
}
|
||||
@@ -445,3 +471,10 @@ func activeHCAs(hcas []discovery.HCA) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if s == "" {
|
||||
return "no answer from the subnet manager"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -24,7 +24,8 @@ func healthy() Facts {
|
||||
f.GPUs = append(f.GPUs, g)
|
||||
}
|
||||
f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE",
|
||||
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1}}}}
|
||||
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
|
||||
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
|
||||
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
||||
return f
|
||||
}
|
||||
@@ -57,6 +58,7 @@ func TestAHealthyNodeHasNothingToFix(t *testing.T) {
|
||||
has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]")
|
||||
has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s")
|
||||
has(t, fs, OK, "NCCL bootstraps over ens3")
|
||||
has(t, fs, OK, "subnet manager 0x98039b03009fcfd6")
|
||||
}
|
||||
|
||||
func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -24,6 +25,9 @@ type DashboardData struct {
|
||||
Unplaced []adminapi.ExplainView
|
||||
Usage []store.KeyUsage
|
||||
Summary adminapi.Summary
|
||||
// Fabrics is the physical fabrics and what disagrees about them, from a
|
||||
// source that can say.
|
||||
Fabrics *fabrics.Report
|
||||
|
||||
Users []store.UserUsage
|
||||
Models2 []store.ModelUsage
|
||||
@@ -91,6 +95,12 @@ type Source interface {
|
||||
Summary() adminapi.Summary
|
||||
}
|
||||
|
||||
// FabricSource is a Source that can say which physical fabrics the adapters
|
||||
// are on. Optional, so a source without one still draws the page.
|
||||
type FabricSource interface {
|
||||
Fabrics() fabrics.Report
|
||||
}
|
||||
|
||||
// UsageSource is the accounting half, absent when authentication is off.
|
||||
type UsageSource interface {
|
||||
UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error)
|
||||
@@ -165,6 +175,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
}
|
||||
d.Unplaced = unplaced(src, d.Models)
|
||||
d.Summary = src.Summary()
|
||||
if fs, ok := src.(FabricSource); ok {
|
||||
r := fs.Fabrics()
|
||||
d.Fabrics = &r
|
||||
}
|
||||
}
|
||||
if usage != nil {
|
||||
rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour))
|
||||
|
||||
@@ -548,3 +548,6 @@ table.held tr.model-group > td { background: var(--bg); padding-top: .7rem; bord
|
||||
table.held tr.model-group strong { font-size: .95rem; }
|
||||
table.held tr.model-version > td:first-child { padding-left: 1.4rem; }
|
||||
.tag.bad { color: var(--bad); border-color: var(--bad); }
|
||||
ul.fabric-problems { list-style: none; padding: 0; margin: .25rem 0 .75rem; font-size: .88rem; }
|
||||
ul.fabric-problems li { margin: .25rem 0; }
|
||||
ul.fabric-problems li.info { color: var(--muted); }
|
||||
@@ -130,18 +130,26 @@
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
|
||||
{{if .Summary.Fabrics}}
|
||||
{{if or .Summary.Fabrics .Fabrics}}
|
||||
<h2>Fabrics</h2>
|
||||
<table><tr><th>Fabric</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}<tr>
|
||||
{{with .Fabrics}}{{if .Problems}}
|
||||
<ul class="fabric-problems">{{range .Problems}}
|
||||
<li class="{{.Level}}"><span class="tag{{if eq (printf "%s" .Level) "fail"}} bad{{end}}">{{.Level}}</span> {{.Message}}</li>{{end}}
|
||||
</ul>{{end}}{{end}}
|
||||
{{if .Summary.Fabrics}}
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th class="num">Nodes</th><th class="num">Fully free</th><th>Weights</th></tr>
|
||||
{{range .Summary.Fabrics}}{{$name := .Fabric}}<tr>
|
||||
<td><code>{{.Fabric}}</code></td>
|
||||
<td>{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}<div>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</div>{{end}}{{end}}{{end}}</td>
|
||||
<td class="num">{{.Nodes}}</td>
|
||||
<td class="num">{{.LargestFreeBlock}}</td>
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
<p class="note">Fully free counts whole unused nodes, which is what decides whether a
|
||||
replica spanning nodes can land: free cards on the wrong fabric are not capacity
|
||||
for a model that needs this one.</p>
|
||||
{{end}}
|
||||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||||
unused nodes, which is what decides whether such a replica can land.</p>
|
||||
{{end}}
|
||||
|
||||
{{with .Fleet}}
|
||||
|
||||
@@ -35,12 +35,13 @@
|
||||
<h2>Fabric</h2>
|
||||
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
|
||||
{{if .Node.IBPorts}}
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
{{range .Node.IBPorts}}<tr>
|
||||
<td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
|
||||
<td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
|
||||
<td class="num">{{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}}</td>
|
||||
<td>{{or .Fabric "—"}}</td>
|
||||
<td>{{if .Physical}}{{.Physical}}{{else if .SMError}}<span class="note" title="{{.SMError}}">unknown</span>{{else}}—{{end}}</td>
|
||||
<td>{{if .Errors}}{{range $k, $v := .Errors}}<span class="tag bad">{{$k}} {{$v}}</span> {{end}}{{else}}<span class="note">none</span>{{end}}</td>
|
||||
<td class="num">{{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}</td>
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"html/template"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -103,6 +104,7 @@ var funcs = template.FuncMap{
|
||||
// about the size of the same file starts an argument about neither.
|
||||
"bytes": humanBytes,
|
||||
"u64": func(v uint64) int64 { return int64(min(v, 1<<62)) },
|
||||
"has": func(list []string, v string) bool { return slices.Contains(list, v) },
|
||||
// dict lets one chart template be called with named arguments. Go
|
||||
// templates pass a single value, and four charts differing in three
|
||||
// strings each is four copies of the same markup otherwise.
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -649,3 +650,23 @@ func TestTheClusterPageLeadsWithHealthAndRefreshesInPlace(t *testing.T) {
|
||||
t.Error("without script the page should still refresh")
|
||||
}
|
||||
}
|
||||
|
||||
// fabricSource is fakeSource that can say what its fabrics physically are.
|
||||
type fabricSource struct{ fakeSource }
|
||||
|
||||
func (fabricSource) Fabrics() fabrics.Report {
|
||||
return fabrics.Report{
|
||||
Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)",
|
||||
Declared: []string{"ib-a"}, RateGbps: 400}},
|
||||
Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) {
|
||||
_, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user