package discovery import ( "fmt" "sort" "strings" ) // FabricCheck is one declared fabric measured against what the node actually // has. type FabricCheck struct { Fabric string // Declared are the adapters the configuration names for it. Declared []string // Usable are the declared adapters that exist and have an active port. Usable []string // RateGbps is the slowest active port among them, because a collective // runs at the speed of its slowest link. RateGbps int // Problems are the declared adapters that cannot carry traffic, and why. Problems []string } // OK reports whether this fabric can carry a collective at all. func (c FabricCheck) OK() bool { return len(c.Usable) > 0 } // CheckFabrics measures declared fabric membership against discovered adapters. // // Fabric membership has to be configuration — OpenSM's default subnet prefix is // the same on every fabric, so a matching prefix proves nothing — but taking it // entirely on trust was a mistake of a different kind. A typo, or a card that // enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter // that does not exist; and an adapter whose cable is out, or whose subnet // manager is down, is present in sysfs and carries nothing. // // Either way the replica starts, passes its health checks, and hangs on its // first all-reduce — which is the failure the whole fabric invariant exists to // prevent, arriving through a door nobody was watching. Declared and real are // now compared where both are visible, which is only here. func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck { byName := make(map[string]HCA, len(discovered)) for _, h := range discovered { byName[h.Name] = h } names := make([]string, 0, len(declared)) for f := range declared { names = append(names, f) } sort.Strings(names) out := make([]FabricCheck, 0, len(names)) for _, fabric := range names { c := FabricCheck{Fabric: fabric, Declared: declared[fabric]} for _, want := range declared[fabric] { hca, found := byName[want] if !found { c.Problems = append(c.Problems, fmt.Sprintf( "%s is not on this node (found: %s)", want, listNames(discovered))) continue } rate, active := activeRate(hca) if !active { c.Problems = append(c.Problems, fmt.Sprintf( "%s has no active port (%s)", want, portStates(hca))) continue } c.Usable = append(c.Usable, want) if c.RateGbps == 0 || rate < c.RateGbps { c.RateGbps = rate } } out = append(out, c) } return out } // activeRate returns the fastest active port's rate on an adapter. // // Only ACTIVE ports carry traffic. A port that is physically linked but has no // subnet manager sits in INIT, looks healthy to anything counting adapters, and // passes no data — which is precisely the case worth catching, because it // survives a reboot and looks like a software problem. func activeRate(h HCA) (int, bool) { best, ok := 0, false for _, p := range h.Ports { if !strings.EqualFold(p.State, "ACTIVE") { continue } ok = true if p.RateGbps > best { best = p.RateGbps } } return best, ok } func portStates(h HCA) string { if len(h.Ports) == 0 { return "no ports" } var parts []string for _, p := range h.Ports { parts = append(parts, fmt.Sprintf("port %d %s/%s", p.Num, p.State, p.PhysState)) } return strings.Join(parts, ", ") } func listNames(hcas []HCA) string { if len(hcas) == 0 { return "none" } var names []string for _, h := range hcas { names = append(names, h.Name) } return strings.Join(names, ", ") }