package discovery import ( "context" "errors" "fmt" "os/exec" "regexp" "strconv" "strings" "time" ) // ErrNoDiags means infiniband-diags is not installed, so the subnet manager // cannot be asked who it is. var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags") // smLine is sminfo's one line of answer: // // sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`) // ParseSMInfo reads sminfo's output. func ParseSMInfo(out string) (*SMInfo, error) { m := smLine.FindStringSubmatch(out) if m == nil { return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out)) } lid, _ := strconv.Atoi(m[1]) prio, _ := strconv.Atoi(m[3]) return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil } // QuerySM asks, through one port, which subnet manager is master. It is a // management datagram to the SM, not a scan: cheap, but not free, which is // why the agent asks once a minute rather than every heartbeat. func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) { bin, err := exec.LookPath("sminfo") if err != nil { return nil, ErrNoDiags } ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput() if err != nil { return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out))) } return ParseSMInfo(string(out)) } // AttachSM fills in the subnet manager each active InfiniBand port sees. // RoCE has none. The first error is returned for reporting; the ports it // could answer for keep their answer. func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error { var first error for i := range hcas { for j := range hcas[i].Ports { p := &hcas[i].Ports[j] if !p.Active() || p.LinkLayer == LinkEthernet { continue } sm, err := query(hcas[i].Name, p.Num) if err != nil { if first == nil { first = err } continue } p.SM = sm } } return first } // SMRecord is one subnet manager on a fabric, as the subnet administrator // lists them: master and standbys alike. type SMRecord struct { GUID string `json:"guid"` LID int `json:"lid"` Priority int `json:"priority"` // State is master, standby, discovering or not active. State string `json:"state"` } // smStates are SMInfo's state numbers. var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"} // smirField is one line of saquery's record dump: "LID.........1". var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`) // ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet // administrator knows of, one "SMInfoRecord dump:" block each. func ParseSMIR(out string) ([]SMRecord, error) { var recs []SMRecord var cur *SMRecord for _, line := range strings.Split(out, "\n") { if strings.Contains(line, "SMInfoRecord dump") { recs = append(recs, SMRecord{}) cur = &recs[len(recs)-1] continue } m := smirField.FindStringSubmatch(line) if m == nil || cur == nil { continue } switch strings.ToLower(m[1]) { case "lid": cur.LID, _ = strconv.Atoi(m[2]) case "guid": cur.GUID = normalizeGUID(m[2]) case "priority": cur.Priority, _ = strconv.Atoi(m[2]) case "smstate": cur.State = smStates[m[2]] if cur.State == "" { cur.State = m[2] } } } out2 := recs[:0] for _, r := range recs { if r.GUID != "" { out2 = append(out2, r) } } if len(out2) == 0 { return nil, fmt.Errorf("no subnet manager records in saquery output") } return out2, nil } // QuerySMs asks the subnet administrator, through one port, for every subnet // manager on the fabric. func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) { bin, err := exec.LookPath("saquery") if err != nil { return nil, ErrNoDiags } ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput() if err != nil { return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out))) } return ParseSMIR(string(out)) } // ScanFabric maps the fabric a port is on with ibnetdiscover: every switch, // adapter and link. It can take a while on a large fabric, so it is bounded, // and its output is capped: a fabric of thousands of ports is a few // megabytes of text. func ScanFabric(ctx context.Context, hca string, port int) (string, error) { bin, err := exec.LookPath("ibnetdiscover") if err != nil { return "", ErrNoDiags } ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).Output() if err != nil { var ee *exec.ExitError if errors.As(err, &ee) { return "", fmt.Errorf("ibnetdiscover on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(ee.Stderr))) } return "", fmt.Errorf("ibnetdiscover on %s/%d: %w", hca, port, err) } const limit = 16 << 20 if len(out) > limit { return "", fmt.Errorf("ibnetdiscover printed %d MB, more than the %d MB a scan may send", len(out)>>20, limit>>20) } return string(out), nil }