llmgw/internal/agent/discovery/sminfo.go

package discovery

import (
	"context"
	"errors"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
// cannot be asked who it is.
var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")

// smLine is sminfo's one line of answer:
//
//	sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)

// ParseSMInfo reads sminfo's output.
func ParseSMInfo(out string) (*SMInfo, error) {
	m := smLine.FindStringSubmatch(out)
	if m == nil {
		return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
	}
	lid, _ := strconv.Atoi(m[1])
	prio, _ := strconv.Atoi(m[3])
	return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
}

// QuerySM asks, through one port, which subnet manager is master. It is a
// management datagram to the SM, not a scan: cheap, but not free, which is
// why the agent asks once a minute rather than every heartbeat.
func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
	bin, err := exec.LookPath("sminfo")
	if err != nil {
		return nil, ErrNoDiags
	}
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
	}
	return ParseSMInfo(string(out))
}

// AttachSM fills in the subnet manager each active InfiniBand port sees.
// RoCE has none. The first error is returned for reporting; the ports it
// could answer for keep their answer.
func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
	var first error
	for i := range hcas {
		for j := range hcas[i].Ports {
			p := &hcas[i].Ports[j]
			if !p.Active() || p.LinkLayer == LinkEthernet {
				continue
			}
			sm, err := query(hcas[i].Name, p.Num)
			if err != nil {
				if first == nil {
					first = err
				}
				continue
			}
			p.SM = sm
		}
	}
	return first
}

// SMRecord is one subnet manager on a fabric, as the subnet administrator
// lists them: master and standbys alike.
type SMRecord struct {
	GUID     string `json:"guid"`
	LID      int    `json:"lid"`
	Priority int    `json:"priority"`
	// State is master, standby, discovering or not active.
	State string `json:"state"`
}

// smStates are SMInfo's state numbers.
var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"}

// smirField is one line of saquery's record dump: "LID.........1".
var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`)

// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet
// administrator knows of, one "SMInfoRecord dump:" block each.
func ParseSMIR(out string) ([]SMRecord, error) {
	var recs []SMRecord
	var cur *SMRecord
	for _, line := range strings.Split(out, "\n") {
		if strings.Contains(line, "SMInfoRecord dump") {
			recs = append(recs, SMRecord{})
			cur = &recs[len(recs)-1]
			continue
		}
		m := smirField.FindStringSubmatch(line)
		if m == nil || cur == nil {
			continue
		}
		switch strings.ToLower(m[1]) {
		case "lid":
			cur.LID, _ = strconv.Atoi(m[2])
		case "guid":
			cur.GUID = normalizeGUID(m[2])
		case "priority":
			cur.Priority, _ = strconv.Atoi(m[2])
		case "smstate":
			cur.State = smStates[m[2]]
			if cur.State == "" {
				cur.State = m[2]
			}
		}
	}
	out2 := recs[:0]
	for _, r := range recs {
		if r.GUID != "" {
			out2 = append(out2, r)
		}
	}
	if len(out2) == 0 {
		return nil, fmt.Errorf("no subnet manager records in saquery output")
	}
	return out2, nil
}

// QuerySMs asks the subnet administrator, through one port, for every subnet
// manager on the fabric.
func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
	bin, err := exec.LookPath("saquery")
	if err != nil {
		return nil, ErrNoDiags
	}
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
	}
	return ParseSMIR(string(out))
}

// ScanFabric maps the fabric a port is on with ibnetdiscover: every switch,
// adapter and link. It can take a while on a large fabric, so it is bounded,
// and its output is capped: a fabric of thousands of ports is a few
// megabytes of text.
func ScanFabric(ctx context.Context, hca string, port int) (string, error) {
	bin, err := exec.LookPath("ibnetdiscover")
	if err != nil {
		return "", ErrNoDiags
	}
	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
	defer cancel()
	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).Output()
	if err != nil {
		var ee *exec.ExitError
		if errors.As(err, &ee) {
			return "", fmt.Errorf("ibnetdiscover on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(ee.Stderr)))
		}
		return "", fmt.Errorf("ibnetdiscover on %s/%d: %w", hca, port, err)
	}
	const limit = 16 << 20
	if len(out) > limit {
		return "", fmt.Errorf("ibnetdiscover printed %d MB, more than the %d MB a scan may send", len(out)>>20, limit>>20)
	}
	return string(out), nil
}