diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index be605db..df937a8 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1398,7 +1398,25 @@ type IBPort struct { // link_layer is InfiniBand or Ethernet (RoCE). LinkLayer string `protobuf:"bytes,6,opt,name=link_layer,json=linkLayer,proto3" json:"link_layer,omitempty"` // counters are cumulative, by sysfs name; data counters in bytes. - Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + // Identity: the adapter's node GUID and the port's own, as the fabric + // knows them. + NodeGuid string `protobuf:"bytes,8,opt,name=node_guid,json=nodeGuid,proto3" json:"node_guid,omitempty"` + PortGuid string `protobuf:"bytes,9,opt,name=port_guid,json=portGuid,proto3" json:"port_guid,omitempty"` + // The subnet manager this port sees as master, on InfiniBand. Every port on + // one physical fabric sees the same one, which is what tells separate + // fabrics apart when their names are only what somebody typed. + SmGuid string `protobuf:"bytes,10,opt,name=sm_guid,json=smGuid,proto3" json:"sm_guid,omitempty"` + SmLid int32 `protobuf:"varint,11,opt,name=sm_lid,json=smLid,proto3" json:"sm_lid,omitempty"` + SmPriority int32 `protobuf:"varint,12,opt,name=sm_priority,json=smPriority,proto3" json:"sm_priority,omitempty"` + SmState string `protobuf:"bytes,13,opt,name=sm_state,json=smState,proto3" json:"sm_state,omitempty"` + // sm_error says why sm_guid is empty on an active InfiniBand port: + // infiniband-diags missing, or no subnet manager answering. + SmError string `protobuf:"bytes,14,opt,name=sm_error,json=smError,proto3" json:"sm_error,omitempty"` + // network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no + // subnet manager, and the network is what its fabric is. + Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"` + GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1482,6 +1500,69 @@ func (x *IBPort) GetCounters() map[string]uint64 { return nil } +func (x *IBPort) GetNodeGuid() string { + if x != nil { + return x.NodeGuid + } + return "" +} + +func (x *IBPort) GetPortGuid() string { + if x != nil { + return x.PortGuid + } + return "" +} + +func (x *IBPort) GetSmGuid() string { + if x != nil { + return x.SmGuid + } + return "" +} + +func (x *IBPort) GetSmLid() int32 { + if x != nil { + return x.SmLid + } + return 0 +} + +func (x *IBPort) GetSmPriority() int32 { + if x != nil { + return x.SmPriority + } + return 0 +} + +func (x *IBPort) GetSmState() string { + if x != nil { + return x.SmState + } + return "" +} + +func (x *IBPort) GetSmError() string { + if x != nil { + return x.SmError + } + return "" +} + +func (x *IBPort) GetNetwork() string { + if x != nil { + return x.Network + } + return "" +} + +func (x *IBPort) GetGidPrefix() string { + if x != nil { + return x.GidPrefix + } + return "" +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -2787,7 +2868,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + - "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x98\x02\n" + + "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x92\x04\n" + "\x06IBPort\x12\x10\n" + "\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" + @@ -2797,7 +2878,19 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\trate_gbps\x18\x05 \x01(\x05R\brateGbps\x12\x1d\n" + "\n" + "link_layer\x18\x06 \x01(\tR\tlinkLayer\x12:\n" + - "\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x1a;\n" + + "\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x12\x1b\n" + + "\tnode_guid\x18\b \x01(\tR\bnodeGuid\x12\x1b\n" + + "\tport_guid\x18\t \x01(\tR\bportGuid\x12\x17\n" + + "\asm_guid\x18\n" + + " \x01(\tR\x06smGuid\x12\x15\n" + + "\x06sm_lid\x18\v \x01(\x05R\x05smLid\x12\x1f\n" + + "\vsm_priority\x18\f \x01(\x05R\n" + + "smPriority\x12\x19\n" + + "\bsm_state\x18\r \x01(\tR\asmState\x12\x19\n" + + "\bsm_error\x18\x0e \x01(\tR\asmError\x12\x18\n" + + "\anetwork\x18\x0f \x01(\tR\anetwork\x12\x1d\n" + + "\n" + + "gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x1a;\n" + "\rCountersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index f435bc3..d9a7874 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -277,6 +277,25 @@ message IBPort { string link_layer = 6; // counters are cumulative, by sysfs name; data counters in bytes. map counters = 7; + + // Identity: the adapter's node GUID and the port's own, as the fabric + // knows them. + string node_guid = 8; + string port_guid = 9; + // The subnet manager this port sees as master, on InfiniBand. Every port on + // one physical fabric sees the same one, which is what tells separate + // fabrics apart when their names are only what somebody typed. + string sm_guid = 10; + int32 sm_lid = 11; + int32 sm_priority = 12; + string sm_state = 13; + // sm_error says why sm_guid is empty on an active InfiniBand port: + // infiniband-diags missing, or no subnet manager answering. + string sm_error = 14; + // network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no + // subnet manager, and the network is what its fabric is. + string network = 15; + string gid_prefix = 16; } message Heartbeat { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index a35c4a5..f659e5e 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -16,6 +16,7 @@ import ( "net/http" "net/url" "os" + "os/exec" "path/filepath" "slices" "strings" @@ -357,6 +358,12 @@ type Agent struct { lastWeights string // fabricsSent is what the control plane was last told about fabrics. fabricsSent string + // sm caches which subnet manager each InfiniBand port sees, asked once + // a minute: a management datagram per port per heartbeat is more than + // an answer that changes on a failover is worth. + sm map[string]*discovery.SMInfo + smErr map[string]string + smAsked time.Time // load measures the machine between heartbeats. load *discovery.LoadSampler @@ -431,6 +438,7 @@ func (a *Agent) Run(ctx context.Context) error { a.log.Warn("cannot create the weights directory", "dir", a.cfg.WeightsCache, "err", err) } go a.provisionLoop(ctx) + go a.ensureDiags(ctx) attempt := 0 next := 0 @@ -647,12 +655,22 @@ func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) { if err != nil { return } + a.attachSM(hcas) for _, h := range hcas { for _, p := range h.Ports { - hb.IbPorts = append(hb.IbPorts, &llmgwv1.IBPort{ + ip := &llmgwv1.IBPort{ Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState, RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters, - }) + NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix, + } + if p.SM != nil { + ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State + } else { + a.mu.Lock() + ip.SmError = a.smErr[smKey(h.Name, p.Num)] + a.mu.Unlock() + } + hb.IbPorts = append(hb.IbPorts, ip) } } a.mu.Lock() @@ -1608,3 +1626,70 @@ func ResolveManagement(c Config) (addr, iface string) { } return addr, iface } + +func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) } + +// querySM is discovery.QuerySM, replaced in tests. +var querySM = func(hca string, port int) (*discovery.SMInfo, error) { + return discovery.QuerySM(context.Background(), hca, port) +} + +// attachSM puts on each active InfiniBand port the subnet manager it sees, +// asking again once a minute and otherwise reusing the last answer. +func (a *Agent) attachSM(hcas []discovery.HCA) { + a.mu.Lock() + stale := time.Since(a.smAsked) > time.Minute || a.sm == nil + a.mu.Unlock() + if stale { + sm, errs := map[string]*discovery.SMInfo{}, map[string]string{} + _ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) { + got, err := querySM(hca, port) + if err != nil { + errs[smKey(hca, port)] = err.Error() + return nil, err + } + sm[smKey(hca, port)] = got + return got, nil + }) + a.mu.Lock() + for k, v := range sm { + if old := a.sm[k]; old != nil && old.GUID != v.GUID { + a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID) + } + } + a.sm, a.smErr, a.smAsked = sm, errs, time.Now() + a.mu.Unlock() + return + } + a.mu.Lock() + defer a.mu.Unlock() + for i := range hcas { + for j := range hcas[i].Ports { + if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() { + hcas[i].Ports[j].SM = sm + } + } + } +} + +// ensureDiags installs infiniband-diags on a node with adapters and without +// it. sminfo is what tells which physical fabric a port is on; without it, +// two separate fabrics given one name cannot be caught. Installed the way +// engine dependencies are, from the machine's own package sources. +func (a *Agent) ensureDiags(ctx context.Context) { + a.mu.Lock() + hasIB := len(a.inv.HCAs) > 0 + a.mu.Unlock() + if !hasIB || a.prov == nil { + return + } + if _, err := exec.LookPath("sminfo"); err == nil { + return + } + a.log.Info("installing infiniband-diags, to tell which fabric each port is on") + pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}} + if err := a.prov.InstallPackages(ctx, pkgs); err != nil { + a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager", + "err", err) + } +} diff --git a/internal/agent/discovery/ib.go b/internal/agent/discovery/ib.go index 7e8fc14..5fdae82 100644 --- a/internal/agent/discovery/ib.go +++ b/internal/agent/discovery/ib.go @@ -5,6 +5,7 @@ package discovery import ( "errors" "fmt" + "net" "os" "path/filepath" "slices" @@ -52,6 +53,40 @@ type Port struct { // default of 0 is a RoCE v1 link-local entry on most setups, which does // not route, and a job on it hangs. RoCEv2GID int `json:"rocev2_gid"` + // PortGUID is the port's own identifier, the interface half of its GID. + PortGUID string `json:"port_guid,omitempty"` + // SM is the subnet manager this port sees as master, from sminfo. Nil on + // RoCE, which has none, and when infiniband-diags is not installed. + SM *SMInfo `json:"sm,omitempty"` + // Network is, on RoCE, the IP network of the RoCE v2 address: the only + // thing that says two RoCE ports are on the same fabric. + Network string `json:"network,omitempty"` +} + +// SMInfo is a subnet manager as a port sees it. +type SMInfo struct { + GUID string `json:"guid"` + LID int `json:"lid"` + Priority int `json:"priority"` + // State is SMINFO_MASTER, SMINFO_STANDBY, ... + State string `json:"state"` +} + +// FabricKey is what identifies the physical fabric this port is on: the +// master subnet manager on InfiniBand, since every port on one fabric sees +// the same one and ports on separate fabrics cannot; the IP network on RoCE. +// Empty when neither is known, and for a port that is not up. +func (p Port) FabricKey() string { + if !p.Active() { + return "" + } + switch { + case p.LinkLayer == LinkEthernet && p.Network != "": + return "roce:" + p.Network + case p.SM != nil && p.SM.GUID != "": + return "sm:" + p.SM.GUID + } + return "" } // PortCounters are the counters worth watching, from each port's counters/ @@ -80,6 +115,9 @@ type HCA struct { // FirmwareVersion is reported by the driver; worth capturing because // NCCL behaviour differs between firmware revisions. FirmwareVersion string `json:"firmware_version,omitempty"` + // NodeGUID identifies the adapter on its fabric: it is the name a + // switch's view of the fabric, and every other tool, knows it by. + NodeGUID string `json:"node_guid,omitempty"` } // ActivePorts returns the ports that can carry traffic. @@ -126,6 +164,7 @@ func DiscoverIB(root string) ([]HCA, error) { func readHCA(dir, name string) (HCA, error) { h := HCA{Name: name, NUMANode: -1} h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver")) + h.NodeGUID = normalizeGUID(readTrimmed(filepath.Join(dir, "node_guid"))) // device is a symlink into the PCI tree; its target's base name is the // bus address, and numa_node lives beside it. @@ -163,8 +202,14 @@ func readPort(dir string, num int) Port { SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))), } p.RoCEv2GID = -1 + p.PortGUID = portGUID(readTrimmed(filepath.Join(dir, "gids", "0"))) if p.LinkLayer == LinkEthernet { p.RoCEv2GID = rocev2GID(dir) + if p.RoCEv2GID >= 0 { + idx := strconv.Itoa(p.RoCEv2GID) + p.Network = RoCENetwork(readTrimmed(filepath.Join(dir, "gid_attrs", "ndevs", idx)), + readTrimmed(filepath.Join(dir, "gids", idx))) + } } for _, name := range PortCounters { raw := readTrimmed(filepath.Join(dir, "counters", name)) @@ -262,3 +307,72 @@ func rocev2GID(dir string) int { } return first } + +// normalizeGUID writes a GUID one way: sysfs says "0c42:a103:0045:bd6c", +// sminfo and ibnetdiscover say "0x0c42a1030045bd6c". +func normalizeGUID(g string) string { + g = strings.ToLower(strings.TrimSpace(g)) + g = strings.TrimPrefix(g, "0x") + g = strings.ReplaceAll(g, ":", "") + if g == "" || strings.Trim(g, "0") == "" { + return "" + } + for len(g) < 16 { + g = "0" + g + } + return "0x" + g +} + +// NormalizeGUID is normalizeGUID for other packages. +func NormalizeGUID(g string) string { return normalizeGUID(g) } + +// portGUID is the interface identifier of a GID: its last four groups. +func portGUID(gid string) string { + parts := strings.Split(gid, ":") + if len(parts) != 8 { + return "" + } + return normalizeGUID(strings.Join(parts[4:], "")) +} + +// RoCENetwork is the IP network an IPv4-mapped RoCE GID's address sits in, +// read from the interface that carries it. Replaced in tests. +var RoCENetwork = func(ndev, gid string) string { + ip := gidIPv4(gid) + if ip == nil || ndev == "" { + return "" + } + ifc, err := net.InterfaceByName(ndev) + if err != nil { + return "" + } + addrs, _ := ifc.Addrs() + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) { + return (&net.IPNet{IP: ip.Mask(n.Mask), Mask: n.Mask}).String() + } + } + return "" +} + +// gidIPv4 is the IPv4 address in an IPv4-mapped GID, nil otherwise. +func gidIPv4(gid string) net.IP { + g := strings.ToLower(gid) + const mapped = "0000:0000:0000:0000:0000:ffff:" + if !strings.HasPrefix(g, mapped) { + return nil + } + hex := strings.ReplaceAll(strings.TrimPrefix(g, mapped), ":", "") + if len(hex) != 8 { + return nil + } + b := make(net.IP, 4) + for i := range 4 { + v, err := strconv.ParseUint(hex[i*2:i*2+2], 16, 8) + if err != nil { + return nil + } + b[i] = byte(v) + } + return b +} diff --git a/internal/agent/discovery/ib_test.go b/internal/agent/discovery/ib_test.go index 884ef66..c8640af 100644 --- a/internal/agent/discovery/ib_test.go +++ b/internal/agent/discovery/ib_test.go @@ -55,6 +55,7 @@ func fakeSysfs(t *testing.T) string { write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2") write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001") write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2") + write("ib/mlx5_2/ports/1/gid_attrs/ndevs/3", "ens1f0") // A PCI device with a NUMA node, linked the way the driver does it. pci := filepath.Join(root, "pci", "0000:0b:00.0") @@ -260,3 +261,65 @@ func TestRoCEFindsItsRoutableGID(t *testing.T) { } } } + +func TestTheSubnetManagerAPortSeesIsRead(t *testing.T) { + sm, err := ParseSMInfo("sminfo: sm lid 1 sm guid 0x98039B03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER\n") + if err != nil { + t.Fatal(err) + } + if sm.GUID != "0x98039b03009fcfd6" || sm.LID != 1 || sm.Priority != 15 || sm.State != "SMINFO_MASTER" { + t.Errorf("sm = %+v", sm) + } + if _, err := ParseSMInfo("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port"); err == nil { + t.Error("an error from sminfo should not parse") + } +} + +func TestAPortsFabricIsItsSubnetManagerOrItsNetwork(t *testing.T) { + // Two ports on one InfiniBand fabric see one master SM, and on separate + // fabrics cannot; RoCE has no SM, and is its IP network. + hcas, err := DiscoverIB(fakeSysfs(t)) + if err != nil { + t.Fatal(err) + } + if hcas[0].NodeGUID != "" { + t.Errorf("no node_guid file, yet %q", hcas[0].NodeGUID) + } + err = AttachSM(hcas, func(hca string, port int) (*SMInfo, error) { + return &SMInfo{GUID: "0x00000000000000aa", LID: 1, State: "SMINFO_MASTER"}, nil + }) + if err != nil { + t.Fatal(err) + } + if got := hcas[0].Ports[0].FabricKey(); got != "sm:0x00000000000000aa" { + t.Errorf("IB key = %q", got) + } + if got := hcas[1].Ports[1].FabricKey(); got != "" { + t.Errorf("a port that is down is on no fabric, got %q", got) + } + if hcas[2].Ports[0].SM != nil { + t.Error("RoCE was asked for a subnet manager") + } + if got := hcas[0].Ports[0].PortGUID; got != "0x0002c90300a01234" { + t.Errorf("port GUID = %q", got) + } +} + +func TestARoCEPortsNetworkIsItsFabric(t *testing.T) { + old := RoCENetwork + t.Cleanup(func() { RoCENetwork = old }) + RoCENetwork = func(ndev, gid string) string { + if ip := gidIPv4(gid); ip != nil && ip.String() == "10.0.0.1" { + return "10.0.0.0/24" + } + return "" + } + hcas, err := DiscoverIB(fakeSysfs(t)) + if err != nil { + t.Fatal(err) + } + roce := hcas[2].Ports[0] + if roce.Network != "10.0.0.0/24" || roce.FabricKey() != "roce:10.0.0.0/24" { + t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey()) + } +} diff --git a/internal/agent/discovery/sminfo.go b/internal/agent/discovery/sminfo.go new file mode 100644 index 0000000..0d0a170 --- /dev/null +++ b/internal/agent/discovery/sminfo.go @@ -0,0 +1,73 @@ +package discovery + +import ( + "context" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +// ErrNoDiags means infiniband-diags is not installed, so the subnet manager +// cannot be asked who it is. +var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags") + +// smLine is sminfo's one line of answer: +// +// sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER +var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`) + +// ParseSMInfo reads sminfo's output. +func ParseSMInfo(out string) (*SMInfo, error) { + m := smLine.FindStringSubmatch(out) + if m == nil { + return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out)) + } + lid, _ := strconv.Atoi(m[1]) + prio, _ := strconv.Atoi(m[3]) + return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil +} + +// QuerySM asks, through one port, which subnet manager is master. It is a +// management datagram to the SM, not a scan: cheap, but not free, which is +// why the agent asks once a minute rather than every heartbeat. +func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) { + bin, err := exec.LookPath("sminfo") + if err != nil { + return nil, ErrNoDiags + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out))) + } + return ParseSMInfo(string(out)) +} + +// AttachSM fills in the subnet manager each active InfiniBand port sees. +// RoCE has none. The first error is returned for reporting; the ports it +// could answer for keep their answer. +func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error { + var first error + for i := range hcas { + for j := range hcas[i].Ports { + p := &hcas[i].Ports[j] + if !p.Active() || p.LinkLayer == LinkEthernet { + continue + } + sm, err := query(hcas[i].Name, p.Num) + if err != nil { + if first == nil { + first = err + } + continue + } + p.SM = sm + } + } + return first +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3bf8632..331c22d 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -10,6 +10,7 @@ import ( "context" "encoding/json" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "io" "net/http" "slices" @@ -48,6 +49,9 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) mux.HandleFunc("GET /api/v1/usage", a.usageByScope) mux.HandleFunc("GET /api/v1/nodes", a.listNodes) + mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Fabrics()) + }) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) // Listing is a read and deleting is a separate verb, so a mistyped @@ -134,6 +138,10 @@ type IBPortView struct { Active bool `json:"active"` // Fabric is the declared fabric this port's adapter carries, if any. Fabric string `json:"fabric,omitempty"` + // Physical is the fabric the port is physically on: its subnet manager, + // or its RoCE network. SMError says why that is not known. + Physical string `json:"physical,omitempty"` + SMError string `json:"sm_error,omitempty"` // Errors are the error counters that are not zero, by name. Errors map[string]uint64 `json:"errors,omitempty"` TxBytes uint64 `json:"tx_bytes,omitempty"` @@ -336,7 +344,13 @@ func (a *API) Nodes() []NodeView { for _, p := range n.IBPorts { pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState, RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA], - TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]} + TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"], SMError: p.SMError} + switch { + case p.RoCE() && p.Network != "": + pv.Physical = "RoCE " + p.Network + case p.SMGUID != "": + pv.Physical = fmt.Sprintf("SM %s (LID %d)", p.SMGUID, p.SMLID) + } for name, c := range p.Counters { if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" { if pv.Errors == nil { @@ -818,3 +832,9 @@ func (a *API) profileChanged(model, profile, was string) bool { } return false } + +// Fabrics is the physical fabrics the adapters are on, against the names +// the nodes gave them. +func (a *API) Fabrics() fabrics.Report { + return fabrics.Analyze(a.ctl.Inventory().Nodes()) +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index edd14e9..02b7343 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -411,6 +411,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(), PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()), LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()), + NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(), + SMGUID: p.GetSmGuid(), SMLID: int(p.GetSmLid()), SMPriority: int(p.GetSmPriority()), + SMState: p.GetSmState(), SMError: p.GetSmError(), Network: p.GetNetwork(), }) } } diff --git a/internal/control/fabrics/fabrics.go b/internal/control/fabrics/fabrics.go new file mode 100644 index 0000000..4af1ca2 --- /dev/null +++ b/internal/control/fabrics/fabrics.go @@ -0,0 +1,243 @@ +// Package fabrics works out which physical fabrics the cluster's adapters +// are on, and where that disagrees with what the nodes were told. +// +// A fabric's name is configuration. What it is physically is what the +// adapters report: on InfiniBand the master subnet manager every port on one +// fabric sees, on RoCE the IP network. Kept apart so a site with several +// independent fabrics can see each, and see at once when two were given one +// name, one was given two, or an adapter is cabled somewhere nobody declared. +package fabrics + +import ( + "fmt" + "slices" + "sort" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// Level is how much a problem matters. +type Level string + +// The levels. +const ( + Fail Level = "fail" + Warn Level = "warn" + Info Level = "info" +) + +// Problem is one disagreement, in words. +type Problem struct { + Level Level `json:"level"` + Fabric string `json:"fabric,omitempty"` + Message string `json:"message"` +} + +// Physical is one fabric as the adapters find it. +type Physical struct { + Key string `json:"key"` + // Kind is infiniband or roce. + Kind string `json:"kind"` + // Label is the key as a person reads it. + Label string `json:"label"` + // SM is the master subnet manager, on InfiniBand. + SM *SM `json:"sm,omitempty"` + // Declared are the names nodes gave this fabric; one, when all is well. + Declared []string `json:"declared"` + Members []Member `json:"members"` + RateGbps int `json:"rate_gbps"` + GIDPrefix string `json:"gid_prefix,omitempty"` +} + +// SM is a subnet manager as the ports see it. +type SM struct { + GUID string `json:"guid"` + LID int `json:"lid"` + Priority int `json:"priority"` + State string `json:"state"` +} + +// Member is one node's adapters on a physical fabric. +type Member struct { + Node string `json:"node"` + HCAs []string `json:"hcas"` + // Declared is the name this node gives these adapters, empty when it + // declares them in no fabric. + Declared string `json:"declared,omitempty"` +} + +// Report is the whole picture. +type Report struct { + Physical []Physical `json:"physical"` + Problems []Problem `json:"problems"` + // Unverified are declared fabrics some of whose nodes cannot say which + // physical fabric they are on, with why. + Unverified map[string][]string `json:"unverified,omitempty"` +} + +// Analyze reads every node's ports. +func Analyze(nodes []inventory.Node) Report { + r := Report{Unverified: map[string][]string{}} + byKey := map[string]*Physical{} + declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes + + for _, n := range nodes { + declOf := map[string]string{} // hca -> declared fabric + for _, f := range n.Fabrics { + for _, h := range f.HCAs { + declOf[h] = f.Fabric + } + } + // Which physical fabric each declared fabric's adapters are on here. + seenDecl := map[string]map[string]bool{} + members := map[string]*Member{} // key -> this node's membership + + for _, p := range n.IBPorts { + key := p.FabricKey() + decl := declOf[p.HCA] + if key == "" { + if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" { + r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError) + } + continue + } + ph := byKey[key] + if ph == nil { + ph = &Physical{Key: key, Kind: "infiniband", GIDPrefix: p.GIDPrefix} + if p.RoCE() { + ph.Kind, ph.Label = "roce", "RoCE "+p.Network + } else { + ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState} + ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID) + } + byKey[key] = ph + } + if p.RateGbps > 0 && (ph.RateGbps == 0 || p.RateGbps < ph.RateGbps) { + ph.RateGbps = p.RateGbps + } + m := members[key] + if m == nil { + m = &Member{Node: n.ID, Declared: decl} + members[key] = m + } + m.HCAs = appendOnce(m.HCAs, p.HCA) + if decl != "" { + if seenDecl[decl] == nil { + seenDecl[decl] = map[string]bool{} + } + seenDecl[decl][key] = true + ph.Declared = appendOnce(ph.Declared, decl) + if declaredKeys[decl] == nil { + declaredKeys[decl] = map[string][]string{} + } + declaredKeys[decl][key] = appendOnce(declaredKeys[decl][key], n.ID) + } else { + r.Problems = append(r.Problems, Problem{Level: Warn, Message: fmt.Sprintf( + "%s: %s is up on %s but declared in no fabric, so llmgw sends nothing over it", + n.ID, p.HCA, ph.Label)}) + } + } + for decl, keys := range seenDecl { + if len(keys) > 1 { + r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( + "%s: the adapters it declares as %s are cabled to %d different fabrics; it is left out of multi-node work on %s", + n.ID, decl, len(keys), decl)}) + } + } + for key, m := range members { + byKey[key].Members = append(byKey[key].Members, *m) + } + } + + for decl, keys := range declaredKeys { + if len(keys) < 2 { + continue + } + var parts []string + for _, k := range sortedKeys(keys) { + parts = append(parts, fmt.Sprintf("%s: %s", byKey[k].Label, strings.Join(keys[k], ", "))) + } + r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( + "%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name", + decl, len(keys), strings.Join(parts, "; "))}) + } + for _, ph := range byKey { + sort.Strings(ph.Declared) + sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node }) + if len(ph.Declared) > 1 { + r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: ph.Declared[0], Message: fmt.Sprintf( + "%s are one fabric (%s): a replica could use all of them, but each name is placed on alone", + strings.Join(ph.Declared, " and "), ph.Label)}) + } + r.Physical = append(r.Physical, *ph) + } + sort.Slice(r.Physical, func(i, j int) bool { + a, b := r.Physical[i], r.Physical[j] + if na, nb := first(a.Declared), first(b.Declared); na != nb { + return na < nb + } + return a.Key < b.Key + }) + + // Separate InfiniBand fabrics on one subnet prefix: harmless while they + // stay apart, and the reason their GIDs cannot tell them apart. + byPrefix := map[string][]string{} + for _, ph := range r.Physical { + if ph.Kind == "infiniband" && ph.GIDPrefix != "" { + byPrefix[ph.GIDPrefix] = append(byPrefix[ph.GIDPrefix], ph.Label) + } + } + for prefix, labels := range byPrefix { + if len(labels) > 1 { + r.Problems = append(r.Problems, Problem{Level: Info, Message: fmt.Sprintf( + "%d separate fabrics share the subnet prefix %s; that is the default, and fine while they stay apart — a distinct subnet_prefix per subnet manager makes their GIDs unique", + len(labels), prefix)}) + } + } + for decl, why := range r.Unverified { + r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf( + "%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))}) + } + + sort.SliceStable(r.Problems, func(i, j int) bool { + if li, lj := rank(r.Problems[i].Level), rank(r.Problems[j].Level); li != lj { + return li < lj + } + return r.Problems[i].Message < r.Problems[j].Message + }) + return r +} + +func rank(l Level) int { + switch l { + case Fail: + return 0 + case Warn: + return 1 + } + return 2 +} + +func first(s []string) string { + if len(s) == 0 { + return "~" // undeclared last + } + return s[0] +} + +func appendOnce(s []string, v string) []string { + if slices.Contains(s, v) { + return s + } + return append(s, v) +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/control/fabrics/fabrics_test.go b/internal/control/fabrics/fabrics_test.go new file mode 100644 index 0000000..ac9a81d --- /dev/null +++ b/internal/control/fabrics/fabrics_test.go @@ -0,0 +1,91 @@ +package fabrics + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func ibNode(id, declared, sm string, hcas ...string) inventory.Node { + n := inventory.Node{ID: id} + if declared != "" { + n.Fabrics = []inventory.FabricAttachment{{Fabric: declared, HCAs: hcas}} + } + for _, h := range hcas { + n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400, + LinkLayer: "InfiniBand", SMGUID: sm, SMLID: 1, GIDPrefix: "fe80:0000:0000:0000"}) + } + return n +} + +func problem(t *testing.T, r Report, l Level, fragment string) { + t.Helper() + for _, p := range r.Problems { + if p.Level == l && strings.Contains(p.Message, fragment) { + return + } + } + t.Errorf("no %s problem containing %q in %+v", l, fragment, r.Problems) +} + +func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) { + r := Analyze([]inventory.Node{ + ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"), + ibNode("b1", "ib-b", "0x02", "mlx5_0"), + }) + if len(r.Physical) != 2 { + t.Fatalf("physical = %+v", r.Physical) + } + if r.Physical[0].Declared[0] != "ib-a" || len(r.Physical[0].Members) != 2 || r.Physical[0].SM.GUID != "0x01" { + t.Errorf("ib-a = %+v", r.Physical[0]) + } + for _, p := range r.Problems { + if p.Level != Info { + t.Errorf("unexpected problem: %+v", p) + } + } + // Both on the default prefix: said, quietly. + problem(t, r, Info, "share the subnet prefix") +} + +func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) { + r := Analyze([]inventory.Node{ + ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"), + }) + problem(t, r, Fail, "ib-a is 2 separate fabrics") +} + +func TestTwoNamesOnOneFabricAreSaid(t *testing.T) { + r := Analyze([]inventory.Node{ + ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"), + }) + problem(t, r, Warn, "ib-a and ib-b are one fabric") +} + +func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) { + n := ibNode("a1", "ib-a", "0x01", "mlx5_0") + n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"}) + problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09") +} + +func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) { + n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1") + n.IBPorts[1].SMGUID = "0x02" + problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics") +} + +func TestANodeThatCannotSayIsReported(t *testing.T) { + n := ibNode("x", "ib-a", "", "mlx5_0") + n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags" + problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags") +} + +func TestRoCEIsItsNetwork(t *testing.T) { + n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}}, + IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}} + r := Analyze([]inventory.Node{n}) + if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" { + t.Errorf("physical = %+v", r.Physical) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 942bd5d..6a63bd2 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -120,7 +120,7 @@ type Node struct { IBPorts []IBPort `json:"-"` } -// IBPort is one port's state and cumulative counters. +// IBPort is one port's state, identity and cumulative counters. type IBPort struct { HCA string Port int @@ -129,11 +129,64 @@ type IBPort struct { RateGbps int LinkLayer string Counters map[string]uint64 + + NodeGUID string + PortGUID string + GIDPrefix string + // SMGUID and the rest are the master subnet manager this port sees; + // SMError says why they are empty on an active InfiniBand port. + SMGUID string + SMLID int + SMPriority int + SMState string + SMError string + // Network is a RoCE port's IP network. + Network string } // Active reports whether the port can carry traffic. func (p IBPort) Active() bool { return strings.EqualFold(p.State, "ACTIVE") } +// RoCE reports whether the port is Ethernet carrying RDMA. +func (p IBPort) RoCE() bool { return strings.EqualFold(p.LinkLayer, "Ethernet") } + +// FabricKey names the physical fabric the port is on: its master subnet +// manager on InfiniBand, its IP network on RoCE. Empty when not known, and +// for a port that is not up. +func (p IBPort) FabricKey() string { + if !p.Active() { + return "" + } + switch { + case p.RoCE() && p.Network != "": + return "roce:" + p.Network + case !p.RoCE() && p.SMGUID != "": + return "sm:" + p.SMGUID + } + return "" +} + +// PhysicalFabric is the physical fabric this node's adapters for a declared +// fabric are on. Known is false when none of them says; ok is false when +// they disagree — two adapters declared as one fabric that are cabled to two. +func (n Node) PhysicalFabric(fabric string) (key string, known, ok bool) { + hcas := n.HCAsFor(fabric) + for _, p := range n.IBPorts { + if !slices.Contains(hcas, p.HCA) { + continue + } + k := p.FabricKey() + if k == "" { + continue + } + if key != "" && k != key { + return "", true, false + } + key = k + } + return key, key != "", true +} + // Load is a node's use of its CPU and memory, and each engine's share. type Load struct { At time.Time diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 8bb557b..1b686a3 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -247,52 +247,113 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej } for _, fabric := range fabrics { - pool := fabric + "/" + p.classLabel() - candidates := s.eligibleNodes(p, fabric, avoid, &rejections) - - if len(candidates) < p.Nodes { - rejections = append(rejections, Rejection{ - Profile: p.Name, Pool: pool, - Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)), - }) - continue - } - - // Prefer the emptiest nodes for multi-node work: a replica spanning - // nodes is the least movable thing in the cluster, so give it the - // cleanest machines rather than squeezing it into the gaps. - sort.SliceStable(candidates, func(i, j int) bool { - return len(candidates[i].free) > len(candidates[j].free) - }) - - assignments := make([]NodeAssignment, 0, p.Nodes) - for rank, c := range candidates[:p.Nodes] { - gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric)) - if gpus == nil { - assignments = nil - rejections = append(rejections, Rejection{ - Profile: p.Name, Pool: pool, Node: c.node.ID, - Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode), - }) - break + all := s.eligibleNodes(p, fabric, avoid, &rejections) + for _, group := range physicalGroups(fabric, all, p, &rejections) { + pool := fabric + "/" + p.classLabel() + if group.key != "" && group.split { + pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel() + } + if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil { + return pl, rejections } - assignments = append(assignments, NodeAssignment{ - NodeID: c.node.ID, - Rank: rank, - GPUIndices: gpus, - // Only this fabric's adapters, never the node's others. - HCAs: c.node.HCAsFor(fabric), - }) } - if assignments == nil { - continue - } - - return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}, rejections } return nil, rejections } +// fabricGroup is the candidates on one physical fabric. +type fabricGroup struct { + key string + nodes []candidate + // split is set when the declared fabric turned out to be several. + split bool +} + +// physicalGroups divides a declared fabric's candidates by the physical +// fabric their adapters are on. +// +// A fabric's name is what an operator typed, and two separate fabrics given +// one name would have had a replica spread across both: it starts, and hangs +// on its first collective, since nothing routes between them. Grouped by the +// subnet manager each node's ports see (by IP network on RoCE), a replica +// stays on one. Nodes that do not say — an agent without infiniband-diags — +// are a group of their own rather than trusted to join one that does. +func physicalGroups(fabric string, cands []candidate, p Profile, rejections *[]Rejection) []fabricGroup { + by := map[string][]candidate{} + for _, c := range cands { + key, _, ok := c.node.PhysicalFabric(fabric) + if !ok { + *rejections = append(*rejections, Rejection{ + Profile: p.Name, Pool: fabric + "/" + p.classLabel(), Node: c.node.ID, + Reason: "its adapters declared as " + fabric + " are cabled to different fabrics", + }) + continue + } + by[key] = append(by[key], c) + } + out := make([]fabricGroup, 0, len(by)) + for k, v := range by { + out = append(out, fabricGroup{key: k, nodes: v, split: len(by) > 1}) + } + // Largest first, then by key, so the choice is the same every time. + sort.Slice(out, func(i, j int) bool { + if len(out[i].nodes) != len(out[j].nodes) { + return len(out[i].nodes) > len(out[j].nodes) + } + return out[i].key < out[j].key + }) + return out +} + +// shortKey is a physical fabric's key as a pool name carries it. +func shortKey(k string) string { + if i := strings.IndexByte(k, ':'); i >= 0 { + k = k[i+1:] + } + if len(k) > 10 && strings.HasPrefix(k, "0x") { + return "sm-" + k[len(k)-6:] + } + return k +} + +// placeOnFabric forms a replica from candidates that share one fabric. +func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []candidate, rejections *[]Rejection) *Placement { + if len(candidates) < p.Nodes { + *rejections = append(*rejections, Rejection{ + Profile: p.Name, Pool: pool, + Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)), + }) + return nil + } + + // Prefer the emptiest nodes for multi-node work: a replica spanning + // nodes is the least movable thing in the cluster, so give it the + // cleanest machines rather than squeezing it into the gaps. + sort.SliceStable(candidates, func(i, j int) bool { + return len(candidates[i].free) > len(candidates[j].free) + }) + + assignments := make([]NodeAssignment, 0, p.Nodes) + for rank, c := range candidates[:p.Nodes] { + gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric)) + if gpus == nil { + *rejections = append(*rejections, Rejection{ + Profile: p.Name, Pool: pool, Node: c.node.ID, + Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode), + }) + return nil + } + assignments = append(assignments, NodeAssignment{ + NodeID: c.node.ID, + Rank: rank, + GPUIndices: gpus, + // Only this fabric's adapters, never the node's others. + HCAs: c.node.HCAsFor(fabric), + }) + } + return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments} +} + // candidate is a node that passed the hard constraints. type candidate struct { node inventory.Node diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 8189d4d..9b91d1d 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -888,3 +888,73 @@ func TestWithoutTopologyItIsTheLowestIndices(t *testing.T) { t.Errorf("chose %v", got) } } + +// onSM puts a node's declared adapters on the fabric a subnet manager runs. +func onSM(n inventory.Node, sm string) inventory.Node { + for _, f := range n.Fabrics { + for _, h := range f.HCAs { + n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", + LinkLayer: "InfiniBand", SMGUID: sm}) + } + } + return n +} + +func TestTwoFabricsGivenOneNameAreNeverSpanned(t *testing.T) { + // Somebody named two separate fabrics ib-a. Each has two free nodes; the + // replica needs three. Spread across both it would start and hang on its + // first collective, since nothing routes between them. + st := store( + onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01"), onSM(node("a2", "h100-80g", 8, "ib-a"), "0x01"), + onSM(node("b1", "h100-80g", 8, "ib-a"), "0x02"), onSM(node("b2", "h100-80g", 8, "ib-a"), "0x02"), + ) + res := New(st).Place([]Profile{{Name: "big", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 3}}, nil) + if res.Placement != nil { + t.Fatalf("placed across two physical fabrics: %+v", res.Placement.Nodes) + } + if !strings.Contains(res.Explain(), "ib-a@") { + t.Errorf("the explanation should name the physical fabrics:\n%s", res.Explain()) + } + + // Two nodes fit on either; they come from one. + res = New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + first := res.Placement.Nodes[0].NodeID[0] + for _, a := range res.Placement.Nodes { + if a.NodeID[0] != first { + t.Errorf("a pair spans fabrics: %+v", res.Placement.Nodes) + } + } +} + +func TestANodeThatDoesNotSayIsNotTrustedToJoin(t *testing.T) { + // Without infiniband-diags a node cannot say which fabric it is on. It is + // kept apart from nodes that can, rather than assumed to be with them. + st := store( + onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01"), + node("u1", "h100-80g", 8, "ib-a"), + ) + if res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil); res.Placement != nil { + t.Errorf("a node of unknown fabric was paired with one of known: %+v", res.Placement.Nodes) + } + // Where no node says, the declared name is all there is, as before. + st = store(node("u1", "h100-80g", 8, "ib-a"), node("u2", "h100-80g", 8, "ib-a")) + if res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil); res.Placement == nil { + t.Errorf("no placement:\n%s", res.Explain()) + } +} + +func TestAdaptersOfOneDeclaredFabricOnTwoAreRefused(t *testing.T) { + n := node("x", "h100-80g", 8, "ib-a") + n.IBPorts = []inventory.IBPort{ + {HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x01"}, + {HCA: "mlx5_1", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x02"}, + } + res := New(store(n, onSM(node("y", "h100-80g", 8, "ib-a"), "0x01"))).Place( + []Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement != nil || !strings.Contains(res.Explain(), "cabled to different fabrics") { + t.Errorf("expected a refusal naming the cabling:\n%s", res.Explain()) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 5e7ecde..2f5a908 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -59,8 +59,10 @@ type Facts struct { NVSwitch bool `json:"nvswitch"` FabricManager string `json:"fabric_manager,omitempty"` - HCAs []discovery.HCA `json:"hcas"` - Modules map[string]bool `json:"modules"` + HCAs []discovery.HCA `json:"hcas"` + // SMErrors say, per port, why the subnet manager could not be asked. + SMErrors map[string]string `json:"sm_errors,omitempty"` + Modules map[string]bool `json:"modules"` // ConfigPath is the agent configuration read, empty when there was none. ConfigPath string `json:"config_path,omitempty"` @@ -126,6 +128,14 @@ func Collect(ctx context.Context, o Options) Facts { root = discovery.SysfsRoot } f.HCAs, _ = discovery.DiscoverIB(root) + f.SMErrors = map[string]string{} + _ = discovery.AttachSM(f.HCAs, func(hca string, port int) (*discovery.SMInfo, error) { + sm, err := discovery.QuerySM(ctx, hca, port) + if err != nil { + f.SMErrors[fmt.Sprintf("%s/%d", hca, port)] = err.Error() + } + return sm, err + }) for _, m := range watchedModules { _, err := os.Stat(filepath.Join("/sys/module", m)) f.Modules[m] = err == nil @@ -295,7 +305,23 @@ func Evaluate(f Facts) []Finding { add(Fail, "fabric", where+" is RoCE with no RoCE v2 GID entry: NCCL would use a v1 entry and hang", "give the interface an IPv4 address, or check the adapter's RoCE mode") case p.Active(): - add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)", where, p.State, p.RateGbps, p.LinkLayer), "") + on := "" + switch { + case p.SM != nil: + on = fmt.Sprintf(", subnet manager %s (LID %d, %s)", p.SM.GUID, p.SM.LID, p.SM.State) + case p.Network != "": + on = ", network " + p.Network + } + add(OK, "fabric", fmt.Sprintf("%s %s at %d Gb/s (%s)%s", where, p.State, p.RateGbps, p.LinkLayer, on), "") + if p.LinkLayer == discovery.LinkInfiniBand && p.SM == nil { + why := f.SMErrors[where] + fix := "" + if strings.Contains(why, "infiniband-diags") { + fix = "install infiniband-diags" + } + add(Warn, "fabric", where+": which fabric it is on cannot be told — "+orUnknown(why)+ + "; without it, separate fabrics given one name cannot be caught", fix) + } if p.LinkLayer == discovery.LinkInfiniBand && p.SMLID == 0 { add(Warn, "fabric", where+" is active but reports no subnet manager LID", "") } @@ -445,3 +471,10 @@ func activeHCAs(hcas []discovery.HCA) []string { } return out } + +func orUnknown(s string) string { + if s == "" { + return "no answer from the subnet manager" + } + return s +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 218c572..d84d778 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -24,7 +24,8 @@ func healthy() Facts { f.GPUs = append(f.GPUs, g) } f.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{{Num: 1, State: "ACTIVE", - PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1}}}} + PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1, + SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}} f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) return f } @@ -57,6 +58,7 @@ func TestAHealthyNodeHasNothingToFix(t *testing.T) { has(t, fs, OK, "NVLink groups: [0 1 2 3 4 5 6 7]") has(t, fs, OK, "fabric ib-a: mlx5_0 usable at 400 Gb/s") has(t, fs, OK, "NCCL bootstraps over ens3") + has(t, fs, OK, "subnet manager 0x98039b03009fcfd6") } func TestTheMistakesThatStopAFabricNodeAreNamed(t *testing.T) { diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 4706627..e2f0b37 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -2,6 +2,7 @@ package web import ( "context" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "html/template" "log/slog" "net/http" @@ -24,6 +25,9 @@ type DashboardData struct { Unplaced []adminapi.ExplainView Usage []store.KeyUsage Summary adminapi.Summary + // Fabrics is the physical fabrics and what disagrees about them, from a + // source that can say. + Fabrics *fabrics.Report Users []store.UserUsage Models2 []store.ModelUsage @@ -91,6 +95,12 @@ type Source interface { Summary() adminapi.Summary } +// FabricSource is a Source that can say which physical fabrics the adapters +// are on. Optional, so a source without one still draws the page. +type FabricSource interface { + Fabrics() fabrics.Report +} + // UsageSource is the accounting half, absent when authentication is off. type UsageSource interface { UsageSince(ctx context.Context, since time.Time) ([]store.KeyUsage, error) @@ -165,6 +175,10 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat } d.Unplaced = unplaced(src, d.Models) d.Summary = src.Summary() + if fs, ok := src.(FabricSource); ok { + r := fs.Fabrics() + d.Fabrics = &r + } } if usage != nil { rows, err := usage.UsageSince(r.Context(), time.Now().Add(-24*time.Hour)) diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 5781db3..d404645 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -548,3 +548,6 @@ table.held tr.model-group > td { background: var(--bg); padding-top: .7rem; bord table.held tr.model-group strong { font-size: .95rem; } table.held tr.model-version > td:first-child { padding-left: 1.4rem; } .tag.bad { color: var(--bad); border-color: var(--bad); } +ul.fabric-problems { list-style: none; padding: 0; margin: .25rem 0 .75rem; font-size: .88rem; } +ul.fabric-problems li { margin: .25rem 0; } +ul.fabric-problems li.info { color: var(--muted); } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index b0e63a6..db8666d 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -130,18 +130,26 @@ {{end}} {{end}} -{{if .Summary.Fabrics}} +{{if or .Summary.Fabrics .Fabrics}}

Fabrics

- -{{range .Summary.Fabrics}} +{{with .Fabrics}}{{if .Problems}} +{{end}}{{end}} +{{if .Summary.Fabrics}} +
FabricNodesFully freeWeights
+{{range .Summary.Fabrics}}{{$name := .Fabric}} + {{end}}
FabricIs physicallyNodesFully freeWeights
{{.Fabric}}{{with $.Fabrics}}{{range .Physical}}{{if has .Declared $name}}
{{.Label}}{{if .RateGbps}} · {{.RateGbps}} Gb/s{{end}}
{{end}}{{end}}{{end}}
{{.Nodes}} {{.LargestFreeBlock}} {{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}not reported{{end}}
-

Fully free counts whole unused nodes, which is what decides whether a -replica spanning nodes can land: free cards on the wrong fabric are not capacity -for a model that needs this one.

+{{end}} +

A fabric's name is what the nodes were told; what it is physically is +the subnet manager its InfiniBand ports see, or the network its RoCE ports are on. +A replica spanning nodes is only ever formed on one of those. Fully free counts whole +unused nodes, which is what decides whether such a replica can land.

{{end}} {{with .Fleet}} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index 0a083ba..389b0f3 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -35,12 +35,13 @@

Fabric

{{if .Node.Fabrics}}On {{join .Node.Fabrics}}.{{else}}On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.{{end}}

{{if .Node.IBPorts}} - +
AdapterStateRateFabricErrors since bootSentReceived
{{range .Node.IBPorts}} + diff --git a/internal/web/web.go b/internal/web/web.go index 48a3377..ab4b114 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -21,6 +21,7 @@ import ( "html/template" "io/fs" "net/http" + "slices" "strings" "time" @@ -103,6 +104,7 @@ var funcs = template.FuncMap{ // about the size of the same file starts an argument about neither. "bytes": humanBytes, "u64": func(v uint64) int64 { return int64(min(v, 1<<62)) }, + "has": func(list []string, v string) bool { return slices.Contains(list, v) }, // dict lets one chart template be called with named arguments. Go // templates pass a single value, and four charts differing in three // strings each is four copies of the same markup otherwise. diff --git a/internal/web/web_test.go b/internal/web/web_test.go index e1caf81..a2838a4 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -13,6 +13,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -649,3 +650,23 @@ func TestTheClusterPageLeadsWithHealthAndRefreshesInPlace(t *testing.T) { t.Error("without script the page should still refresh") } } + +// fabricSource is fakeSource that can say what its fabrics physically are. +type fabricSource struct{ fakeSource } + +func (fabricSource) Fabrics() fabrics.Report { + return fabrics.Report{ + Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)", + Declared: []string{"ib-a"}, RateGbps: 400}}, + Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}}, + } +} + +func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) { + _, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) + for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `fail`} { + if !strings.Contains(body, want) { + t.Errorf("missing %q", want) + } + } +}
AdapterStateRateFabricPhysically onErrors since bootSentReceived
{{.HCA}}/{{.Port}}{{if eq .LinkLayer "Ethernet"}} RoCE{{end}} {{.State}}{{with .PhysState}} {{.}}{{end}} {{if .RateGbps}}{{.RateGbps}} Gb/s{{else}}—{{end}} {{or .Fabric "—"}}{{if .Physical}}{{.Physical}}{{else if .SMError}}unknown{{else}}—{{end}} {{if .Errors}}{{range $k, $v := .Errors}}{{$k}} {{$v}} {{end}}{{else}}none{{end}} {{if .TxBytes}}{{bytes (u64 .TxBytes)}}{{else}}—{{end}} {{if .RxBytes}}{{bytes (u64 .RxBytes)}}{{else}}—{{end}}