llmgw/internal/agent/discovery/fabrics.go

package discovery

import (
	"fmt"
	"sort"
	"strings"
)

// FabricCheck is one declared fabric measured against what the node actually
// has.
type FabricCheck struct {
	Fabric string
	// Declared are the adapters the configuration names for it.
	Declared []string
	// Usable are the declared adapters that exist and have an active port.
	Usable []string
	// RateGbps is the slowest active port among them, because a collective
	// runs at the speed of its slowest link.
	RateGbps int
	// Problems are the declared adapters that cannot carry traffic, and why.
	Problems []string
}

// OK reports whether this fabric can carry a collective at all.
func (c FabricCheck) OK() bool { return len(c.Usable) > 0 }

// CheckFabrics measures declared fabric membership against discovered adapters.
//
// Fabric membership has to be configuration — OpenSM's default subnet prefix is
// the same on every fabric, so a matching prefix proves nothing — but taking it
// entirely on trust was a mistake of a different kind. A typo, or a card that
// enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter
// that does not exist; and an adapter whose cable is out, or whose subnet
// manager is down, is present in sysfs and carries nothing.
//
// Either way the replica starts, passes its health checks, and hangs on its
// first all-reduce — which is the failure the whole fabric invariant exists to
// prevent, arriving through a door nobody was watching. Declared and real are
// now compared where both are visible, which is only here.
func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck {
	byName := make(map[string]HCA, len(discovered))
	for _, h := range discovered {
		byName[h.Name] = h
	}

	names := make([]string, 0, len(declared))
	for f := range declared {
		names = append(names, f)
	}
	sort.Strings(names)

	out := make([]FabricCheck, 0, len(names))
	for _, fabric := range names {
		c := FabricCheck{Fabric: fabric, Declared: declared[fabric]}
		for _, want := range declared[fabric] {
			hca, found := byName[want]
			if !found {
				c.Problems = append(c.Problems, fmt.Sprintf(
					"%s is not on this node (found: %s)", want, listNames(discovered)))
				continue
			}
			rate, active := activeRate(hca)
			if !active {
				c.Problems = append(c.Problems, fmt.Sprintf(
					"%s has no active port (%s)", want, portStates(hca)))
				continue
			}
			c.Usable = append(c.Usable, want)
			if c.RateGbps == 0 || rate < c.RateGbps {
				c.RateGbps = rate
			}
		}
		out = append(out, c)
	}
	return out
}

// activeRate returns the fastest active port's rate on an adapter.
//
// Only ACTIVE ports carry traffic. A port that is physically linked but has no
// subnet manager sits in INIT, looks healthy to anything counting adapters, and
// passes no data — which is precisely the case worth catching, because it
// survives a reboot and looks like a software problem.
func activeRate(h HCA) (int, bool) {
	best, ok := 0, false
	for _, p := range h.Ports {
		if !strings.EqualFold(p.State, "ACTIVE") {
			continue
		}
		ok = true
		if p.RateGbps > best {
			best = p.RateGbps
		}
	}
	return best, ok
}

func portStates(h HCA) string {
	if len(h.Ports) == 0 {
		return "no ports"
	}
	var parts []string
	for _, p := range h.Ports {
		parts = append(parts, fmt.Sprintf("port %d %s/%s", p.Num, p.State, p.PhysState))
	}
	return strings.Join(parts, ", ")
}

func listNames(hcas []HCA) string {
	if len(hcas) == 0 {
		return "none"
	}
	var names []string
	for _, h := range hcas {
		names = append(names, h.Name)
	}
	return strings.Join(names, ", ")
}