vLLM's mp launch gave workers no rank and the leader's address as their own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank, --master-addr and --master-port, and the workers --headless, as vLLM documents it. NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one interface name for every node; the agent already worked out its own - the one holding its management address - and now uses it. On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts, which does not route. The agent finds the RoCE v2 entry holding an IPv4 address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none. A worker whose only process needs the leader - a Ray join - started without waiting for it, since the wait was honoured for sidecars alone; it found no head, exited and failed the replica. It now waits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
265 lines
7.9 KiB
Go
265 lines
7.9 KiB
Go
// Package discovery inspects a node's GPUs, InfiniBand adapters and the
|
|
// topology between them (plan §5.1 and §5.2).
|
|
package discovery
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// SysfsRoot is the default place InfiniBand devices appear.
|
|
const SysfsRoot = "/sys/class/infiniband"
|
|
|
|
// LinkLayer distinguishes real InfiniBand from RoCE.
|
|
type LinkLayer string
|
|
|
|
const (
|
|
// LinkInfiniBand is native InfiniBand.
|
|
LinkInfiniBand LinkLayer = "InfiniBand"
|
|
// LinkEthernet is RoCE — Ethernet carrying RDMA.
|
|
LinkEthernet LinkLayer = "Ethernet"
|
|
)
|
|
|
|
// Port is one port of an HCA.
|
|
type Port struct {
|
|
Num int `json:"num"`
|
|
// State is the port's logical state; only ACTIVE ports carry traffic.
|
|
State string `json:"state"`
|
|
// PhysState is the physical link state, useful for telling "cable unplugged"
|
|
// from "no subnet manager".
|
|
PhysState string `json:"phys_state"`
|
|
// RateGbps is the negotiated rate.
|
|
RateGbps int `json:"rate_gbps"`
|
|
LinkLayer LinkLayer `json:"link_layer"`
|
|
// GIDPrefix is the subnet prefix from gids/0, normalized to lowercase hex
|
|
// groups. It is the only automatic fabric hint available, and an unreliable
|
|
// one: OpenSM's default is fe80:: everywhere (plan §5.2).
|
|
GIDPrefix string `json:"gid_prefix"`
|
|
LID int `json:"lid"`
|
|
SMLID int `json:"sm_lid"`
|
|
// Counters are the port's error and traffic counters, by their sysfs
|
|
// names, as far as the driver has them. The data counters are converted
|
|
// to bytes; sysfs counts them in four-byte words.
|
|
Counters map[string]uint64 `json:"counters,omitempty"`
|
|
// RoCEv2GID is the GID table index NCCL should use on a RoCE port: the
|
|
// first RoCE v2 entry holding an IPv4 address, else the first RoCE v2
|
|
// entry. -1 on InfiniBand, and on a RoCE port with no v2 entry. NCCL's
|
|
// default of 0 is a RoCE v1 link-local entry on most setups, which does
|
|
// not route, and a job on it hangs.
|
|
RoCEv2GID int `json:"rocev2_gid"`
|
|
}
|
|
|
|
// PortCounters are the counters worth watching, from each port's counters/
|
|
// directory. The errors are the ones that say a cable, a transceiver or a
|
|
// switch port is going bad before a job fails on it; the data counters say
|
|
// whether the fabric is carrying anything at all.
|
|
var PortCounters = []string{
|
|
"symbol_error", "link_error_recovery", "link_downed", "port_rcv_errors",
|
|
"port_rcv_remote_physical_errors", "port_xmit_discards",
|
|
"local_link_integrity_errors", "excessive_buffer_overrun_errors",
|
|
"port_xmit_data", "port_rcv_data",
|
|
}
|
|
|
|
// dataCounter reports whether a counter counts data in four-byte words.
|
|
func dataCounter(name string) bool { return name == "port_xmit_data" || name == "port_rcv_data" }
|
|
|
|
// Active reports whether the port can carry traffic.
|
|
func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
|
|
|
|
// HCA is one host channel adapter.
|
|
type HCA struct {
|
|
Name string `json:"name"`
|
|
Ports []Port `json:"ports"`
|
|
PCIAddr string `json:"pci_addr,omitempty"`
|
|
NUMANode int `json:"numa_node"`
|
|
// FirmwareVersion is reported by the driver; worth capturing because
|
|
// NCCL behaviour differs between firmware revisions.
|
|
FirmwareVersion string `json:"firmware_version,omitempty"`
|
|
}
|
|
|
|
// ActivePorts returns the ports that can carry traffic.
|
|
func (h HCA) ActivePorts() []Port {
|
|
var out []Port
|
|
for _, p := range h.Ports {
|
|
if p.Active() {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// DiscoverIB enumerates the InfiniBand adapters under root.
|
|
//
|
|
// A missing root is not an error: plenty of nodes have no IB at all, and the
|
|
// scheduler treats them as single-node islands (plan §2).
|
|
func DiscoverIB(root string) ([]HCA, error) {
|
|
if root == "" {
|
|
root = SysfsRoot
|
|
}
|
|
entries, err := os.ReadDir(root)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return nil, nil
|
|
}
|
|
return nil, fmt.Errorf("reading %s: %w", root, err)
|
|
}
|
|
|
|
var out []HCA
|
|
for _, e := range entries {
|
|
hca, err := readHCA(filepath.Join(root, e.Name()), e.Name())
|
|
if err != nil {
|
|
// One unreadable adapter must not hide the rest; the node may still
|
|
// be usable through its other HCAs.
|
|
continue
|
|
}
|
|
out = append(out, hca)
|
|
}
|
|
slices.SortFunc(out, func(a, b HCA) int { return strings.Compare(a.Name, b.Name) })
|
|
return out, nil
|
|
}
|
|
|
|
func readHCA(dir, name string) (HCA, error) {
|
|
h := HCA{Name: name, NUMANode: -1}
|
|
h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver"))
|
|
|
|
// device is a symlink into the PCI tree; its target's base name is the
|
|
// bus address, and numa_node lives beside it.
|
|
if target, err := filepath.EvalSymlinks(filepath.Join(dir, "device")); err == nil {
|
|
h.PCIAddr = filepath.Base(target)
|
|
if n, err := strconv.Atoi(readTrimmed(filepath.Join(target, "numa_node"))); err == nil {
|
|
h.NUMANode = n
|
|
}
|
|
}
|
|
|
|
portDirs, err := os.ReadDir(filepath.Join(dir, "ports"))
|
|
if err != nil {
|
|
return h, fmt.Errorf("%s: no ports: %w", name, err)
|
|
}
|
|
for _, pd := range portDirs {
|
|
num, err := strconv.Atoi(pd.Name())
|
|
if err != nil {
|
|
continue
|
|
}
|
|
h.Ports = append(h.Ports, readPort(filepath.Join(dir, "ports", pd.Name()), num))
|
|
}
|
|
slices.SortFunc(h.Ports, func(a, b Port) int { return a.Num - b.Num })
|
|
return h, nil
|
|
}
|
|
|
|
func readPort(dir string, num int) Port {
|
|
p := Port{
|
|
Num: num,
|
|
State: parseEnumeratedState(readTrimmed(filepath.Join(dir, "state"))),
|
|
PhysState: parseEnumeratedState(readTrimmed(filepath.Join(dir, "phys_state"))),
|
|
RateGbps: parseRate(readTrimmed(filepath.Join(dir, "rate"))),
|
|
LinkLayer: LinkLayer(readTrimmed(filepath.Join(dir, "link_layer"))),
|
|
GIDPrefix: parseGIDPrefix(readTrimmed(filepath.Join(dir, "gids", "0"))),
|
|
LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
|
|
SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
|
|
}
|
|
p.RoCEv2GID = -1
|
|
if p.LinkLayer == LinkEthernet {
|
|
p.RoCEv2GID = rocev2GID(dir)
|
|
}
|
|
for _, name := range PortCounters {
|
|
raw := readTrimmed(filepath.Join(dir, "counters", name))
|
|
v, err := strconv.ParseUint(raw, 10, 64)
|
|
if err != nil {
|
|
continue // not every driver has every counter
|
|
}
|
|
if dataCounter(name) {
|
|
v *= 4
|
|
}
|
|
if p.Counters == nil {
|
|
p.Counters = map[string]uint64{}
|
|
}
|
|
p.Counters[name] = v
|
|
}
|
|
return p
|
|
}
|
|
|
|
// parseEnumeratedState turns sysfs's "4: ACTIVE" into "ACTIVE".
|
|
func parseEnumeratedState(s string) string {
|
|
if _, rest, found := strings.Cut(s, ":"); found {
|
|
return strings.TrimSpace(rest)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// parseRate turns "100 Gb/sec (4X EDR)" into 100.
|
|
func parseRate(s string) int {
|
|
f := strings.Fields(s)
|
|
if len(f) == 0 {
|
|
return 0
|
|
}
|
|
// The rate can be fractional ("2.5 Gb/sec") on old hardware; truncate.
|
|
v, err := strconv.ParseFloat(f[0], 64)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return int(v)
|
|
}
|
|
|
|
// parseGIDPrefix keeps the first four groups of a GID, which is the subnet
|
|
// prefix; the remaining four are the interface id.
|
|
func parseGIDPrefix(gid string) string {
|
|
parts := strings.Split(strings.ToLower(gid), ":")
|
|
if len(parts) < 4 {
|
|
return ""
|
|
}
|
|
return strings.Join(parts[:4], ":")
|
|
}
|
|
|
|
// parseMaybeHex accepts both "0x1a" and "26"; sysfs is inconsistent between
|
|
// drivers and kernel versions.
|
|
func parseMaybeHex(s string) int {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" {
|
|
return 0
|
|
}
|
|
if after, ok := strings.CutPrefix(strings.ToLower(s), "0x"); ok {
|
|
v, err := strconv.ParseInt(after, 16, 64)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return int(v)
|
|
}
|
|
v, err := strconv.Atoi(s)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return v
|
|
}
|
|
|
|
func readTrimmed(path string) string {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(string(b))
|
|
}
|
|
|
|
// rocev2GID finds the GID index a RoCE port's traffic should use.
|
|
func rocev2GID(dir string) int {
|
|
first := -1
|
|
for i := 0; i < 64; i++ {
|
|
typ := readTrimmed(filepath.Join(dir, "gid_attrs", "types", strconv.Itoa(i)))
|
|
if !strings.EqualFold(typ, "RoCE v2") {
|
|
continue
|
|
}
|
|
gid := strings.ToLower(readTrimmed(filepath.Join(dir, "gids", strconv.Itoa(i))))
|
|
if strings.HasPrefix(gid, "0000:0000:0000:0000:0000:ffff:") {
|
|
return i
|
|
}
|
|
if first < 0 {
|
|
first = i
|
|
}
|
|
}
|
|
return first
|
|
}
|