rollcall/cmd/rollcall/ipmi.go

package main

// A machine's BMC: the record commissioning (or an operator) leaves in
// machines.bmc, the sealed password, and the API that sets and probes it.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"
)

// bmcCreds is what a power driver needs.
type bmcCreds struct {
	Driver   string            `json:"driver"`
	Address  string            `json:"address"`
	Port     int               `json:"port"`
	User     string            `json:"user"`
	Password string            `json:"password"`
	Extra    map[string]string `json:"extra"`
}

var bootDevices = map[string]bool{"pxe": true, "disk": true, "cdrom": true, "bios": true, "none": true, "safe": true}

type bmcRecord struct {
	Driver  string            `json:"driver"`
	Address string            `json:"address"`
	Port    int               `json:"port"`
	User    string            `json:"user"`
	Extra   map[string]string `json:"extra"`
}

func (a *App) credsFor(m *Machine) (*bmcCreds, error) {
	if m.BMC == nil || m.bmcSecret == nil {
		return nil, fmt.Errorf("no BMC credentials stored")
	}
	var b bmcRecord
	if err := json.Unmarshal(m.BMC, &b); err != nil {
		return nil, err
	}
	if b.Driver == "manual" {
		return &bmcCreds{Driver: "manual"}, nil
	}
	if b.Address == "" {
		return nil, fmt.Errorf("BMC record has no address")
	}
	pw, err := a.sealer.Open(m.bmcSecret, m.ID)
	if err != nil {
		return nil, fmt.Errorf("decrypt BMC secret: %w", err)
	}
	if b.Driver == "" {
		b.Driver = "ipmi"
	}
	if b.Extra == nil {
		b.Extra = map[string]string{}
	}
	return &bmcCreds{Driver: b.Driver, Address: b.Address, Port: b.Port, User: b.User, Password: pw, Extra: b.Extra}, nil
}

var errNoBMC = errors.New("no BMC credentials stored")

// power runs a power operation for a machine on the rack controller that can reach its BMC.
func (a *App) power(ctx context.Context, m *Machine, op, arg string) (string, error) {
	c, err := a.credsFor(m)
	if err != nil {
		return "", fmt.Errorf("%w: %v", errNoBMC, err)
	}
	return a.powerVia(ctx, a.rackOf(m), c, op, arg)
}

func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg string) (string, error) {
	if rack == a.cfg.RackName || rack == "" {
		return powerRun(ctx, a.cfg.IPMITool, c, op, arg)
	}
	r, ok := a.racksSnapshot()[rack]
	if !ok || !rackOnline(r, a.cfg.RackName) {
		return "", fmt.Errorf("rack controller %s is offline", rack)
	}
	return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User,
		Password: c.Password, Extra: c.Extra, Op: op, Arg: arg})
}

// firmwareVia runs a firmware operation (fw-*) where the BMC is reachable.
func (a *App) firmwareVia(ctx context.Context, m *Machine, op, arg string) (string, error) {
	c, err := a.credsFor(m)
	if err != nil {
		return "", fmt.Errorf("%w: %v", errNoBMC, err)
	}
	rack := a.rackOf(m)
	if rack == a.cfg.RackName || rack == "" {
		return powerRun(ctx, a.cfg.IPMITool, c, op, arg)
	}
	r, ok := a.racksSnapshot()[rack]
	if !ok || !rackOnline(r, a.cfg.RackName) {
		return "", fmt.Errorf("rack controller %s is offline", rack)
	}
	return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User,
		Password: c.Password, Extra: c.Extra, Op: op, Arg: arg, Timeout: int((fwTimeouts[op] + time.Minute).Seconds())})
}

func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error {
	if secret != nil {
		_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret)
		return err
	}
	_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2 WHERE id=$1`, id, bmc)
	return err
}

// probeBMC: after commissioning, a BMC still on plain IPMI is asked whether it
// speaks Redfish with the same account; if it does, the vendor's driver takes over.
// An operator's explicit choice (driver_set) is left alone.
// verifyHintedBMC runs a status call against a BMC from bmc_hints (the agent no
// longer gets its password) and records whether it answered.
func (a *App) verifyHintedBMC(m *Machine) {
	var b obj
	if m.BMC == nil || json.Unmarshal(m.BMC, &b) != nil || str(b["source"]) != "hint" || b["driver_set"] == true {
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
	defer cancel()
	out, err := a.power(ctx, m, "status", "")
	b["verified_oob"] = err == nil
	if err != nil {
		b["errors"] = []string{"hinted BMC did not answer: " + oneLine(err.Error())}
	} else {
		delete(b, "errors")
	}
	nb, _ := json.Marshal(b)
	_ = a.store.SetBMC(ctx, m.ID, nb, nil)
	_ = a.store.AddEvent(ctx, m.ID, "bmc hint", fmt.Sprintf("%s: %s", str(b["address"]), oneLine(orDefault(out, fmt.Sprint(err)))))
}

func (a *App) probeBMC(m *Machine) {
	if m.BMC == nil || m.bmcSecret == nil {
		return
	}
	var b obj
	if json.Unmarshal(m.BMC, &b) != nil || b["driver_set"] == true || str(b["address"]) == "" {
		return
	}
	if d := str(b["driver"]); d != "" && d != "ipmi" {
		return
	}
	c, err := a.credsFor(m)
	if err != nil {
		return
	}
	probe := *c
	probe.Driver, probe.Port = "redfish", 0
	ctx, cancel := context.WithTimeout(context.Background(), 90e9)
	defer cancel()
	out, err := a.powerVia(ctx, a.rackOf(m), &probe, "probe", "")
	if err != nil {
		_ = a.store.AddEvent(ctx, m.ID, "bmc probe", "staying on ipmi: "+oneLine(err.Error()))
		return
	}
	var res obj
	if json.Unmarshal([]byte(out), &res) != nil || str(res["driver"]) == "" {
		return
	}
	extra, _ := b["extra"].(map[string]any)
	if extra == nil {
		extra = map[string]any{}
	}
	extra["system"] = str(res["system"])
	b["driver"], b["extra"], b["redfish"] = str(res["driver"]), extra, res
	nb, _ := json.Marshal(b)
	if err := a.store.SetBMC(ctx, m.ID, nb, nil); err != nil {
		log.Printf("probe %s: %v", m.ID, err)
		return
	}
	_ = a.store.AddEvent(ctx, m.ID, "bmc driver", fmt.Sprintf("%s (Redfish %s, %s %s)", str(res["driver"]), str(res["redfish"]), str(res["manufacturer"]), str(res["model"])))
	log.Printf("%s: BMC speaks Redfish %s (%s) -> driver %s", m.ID, str(res["redfish"]), str(res["vendor"]), str(res["driver"]))
}

// handleBMC: GET shows the record (no secret); POST sets driver, address,
// account and extra settings, checked with a status call first when verify is set.
func (a *App) handleBMC(w http.ResponseWriter, r *http.Request) {
	m, err := a.store.Get(r.Context(), r.PathValue("id"))
	if err != nil {
		fail(w, 500, err)
		return
	}
	var b obj
	_ = json.Unmarshal(m.BMC, &b)
	if b == nil {
		b = obj{}
	}
	if r.Method == http.MethodGet {
		writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "has_secret": m.bmcSecret != nil, "drivers": powerDrivers})
		return
	}
	var req struct {
		Driver   *string           `json:"driver"`
		Address  *string           `json:"address"`
		Port     *int              `json:"port"`
		User     *string           `json:"user"`
		Password *string           `json:"password"`
		Extra    map[string]string `json:"extra"`
		Verify   bool              `json:"verify"`
		Probe    bool              `json:"probe"`
	}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		fail(w, 400, err)
		return
	}
	if req.Probe { // re-run Redfish detection
		b["driver_set"] = false
		b["driver"] = "ipmi"
		nb, _ := json.Marshal(b)
		_ = a.store.SetBMC(r.Context(), m.ID, nb, nil)
		if m, err = a.store.Get(r.Context(), m.ID); err != nil {
			fail(w, 500, err)
			return
		}
		a.probeBMC(m)
		if m, err = a.store.Get(r.Context(), m.ID); err != nil {
			fail(w, 500, err)
			return
		}
		_ = json.Unmarshal(m.BMC, &b)
		writeJSON(w, 200, obj{"id": m.ID, "bmc": b})
		return
	}
	if req.Driver != nil {
		if _, ok := driverByName(*req.Driver); !ok {
			fail(w, 400, fmt.Errorf("unknown driver %q", *req.Driver))
			return
		}
		b["driver"] = *req.Driver
	}
	if req.Address != nil {
		b["address"] = strings.TrimSpace(*req.Address)
	}
	if req.Port != nil {
		b["port"] = *req.Port
	}
	if req.User != nil {
		b["user"] = *req.User
	}
	if req.Extra != nil {
		ex, _ := b["extra"].(map[string]any)
		if ex == nil {
			ex = map[string]any{}
		}
		for k, v := range req.Extra {
			if v == "" {
				delete(ex, k)
			} else {
				ex[k] = v
			}
		}
		b["extra"] = ex
	}
	b["driver_set"] = true
	if str(b["source"]) == "" {
		b["source"] = "operator"
	}
	secret := m.bmcSecret
	if req.Password != nil {
		if secret, err = a.sealer.Seal(*req.Password, m.ID); err != nil {
			fail(w, 500, err)
			return
		}
	}
	nb, _ := json.Marshal(b)
	trial := *m
	trial.BMC, trial.bmcSecret = nb, secret
	out := ""
	if req.Verify {
		out, err = a.power(r.Context(), &trial, "status", "")
		if err != nil {
			fail(w, 409, fmt.Errorf("not saved, the BMC did not answer: %v %s", err, out))
			return
		}
		b["configured"], b["verified_oob"] = true, true
		nb, _ = json.Marshal(b)
	}
	if err := a.store.SetBMC(r.Context(), m.ID, nb, secret); err != nil {
		fail(w, 500, err)
		return
	}
	_ = a.store.AddEvent(r.Context(), m.ID, "bmc set", fmt.Sprintf("driver=%s address=%s user=%s", str(b["driver"]), str(b["address"]), str(b["user"])))
	writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "output": out})
}