Firmware updates over Redfish, per vendor

A firmware repository on the region (firmware_dir, served under
/firmware/<id>/<file>, cached by rack controllers for BMCs to pull),
each machine's firmware inventory (FirmwareInventory, classified into
bios/bmc/nic/raid/cpld/psu/drive), and update jobs (kind "firmware")
that the region drives with short rack commands: fw-start hands the
package over, fw-poll watches the task (tolerating a BMC that restarts),
fw-finish releases what was claimed, fw-inventory verifies the version.
Running updates resume after a region restart; they cannot be cancelled
once the BMC has the package.

Standard methods (multipart, SimpleUpdate pull, HttpPushUri) with known
Content-Length, plus vendor plans from their documentation or their own
tools: Dell MultipartUpload (OnReset), HPE SimpleUpdate with .fwpkg
extracted at import, Lenovo /mfwupdate with HttpPushUriTargetsBusy,
Supermicro /upload with targets and preserve options, GIGABYTE/AMI/
ASRock Rack /upload with OemParameters ImageType, OpenBMC
update-multipart, xFusion upload to /tmp/web then SimpleUpdate, KAYTUS
upload then OEM SimpleUpdate with fixed tasks, Fujitsu iRMC OEM actions;
Cisco CIMC is refused clearly (SCP only). fw_params and fw_method on
the BMC record override a plan.

CLI rollcall firmware list|import|rm|show|update, API, a Firmware tab
and a repository card in the UI. The lab Redfish emulator gains an
UpdateService (BMC applies at once, BIOS at the next reset).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/rollcall/config.go

@@ -31,8 +31,10 @@ type Config struct {
 	ClusterVars map[string]any    `json:"cluster_vars"` // available to overlay templates as .Vars (profile vars override)
 
 	// disk-based deployment
-	OSDir     string `json:"os_dir"`     // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
-	OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
+	OSDir string `json:"os_dir"` // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
+	// FirmwareDir: <firmware_dir>/<id>/{meta.json,<package>}, served under /firmware/<id>/<package>
+	FirmwareDir string `json:"firmware_dir"`
+	OSSources   string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
 
 	// region/rack: this server runs an embedded rack controller named RackName;
 	// remote rack controllers (rollcall rack) authenticate with RackSecret
@@ -103,6 +105,9 @@ func loadConfig(path string) (*Config, error) {
 	if c.OSDir == "" {
 		c.OSDir = filepath.Join(filepath.Dir(c.ImageDir), "os")
 	}
+	if c.FirmwareDir == "" {
+		c.FirmwareDir = filepath.Join(filepath.Dir(c.ImageDir), "firmware")
+	}
 	if c.OSSources == "" {
 		c.OSSources = "/etc/rollcall/os-sources.json"
 	}

cmd/rollcall/deploy.go

@@ -626,6 +626,10 @@ func (a *App) handleJobCancel(w http.ResponseWriter, r *http.Request) {
 		fail(w, 404, fmt.Errorf("no active job"))
 		return
 	}
+	if j.Kind == "firmware" {
+		fail(w, 409, errors.New("a firmware update cannot be stopped from here once the BMC has it (it would leave the component half-flashed); it ends by itself"))
+		return
+	}
 	_ = a.store.FinishJob(r.Context(), j.ID, "cancelled", "cancelled by operator")
 	a.afterJob(r.Context(), m, j, "cancelled", "cancelled by operator")
 	writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "state": "cancelled"})
@@ -792,7 +796,7 @@ func (a *App) jobStartTimeout() time.Duration {
 func staleJobs(active map[string]*Job, cutoff time.Time) []*Job {
 	var out []*Job
 	for _, j := range active {
-		if j.State == "pending" && j.Created.Before(cutoff) {
+		if j.State == "pending" && j.Kind != "firmware" && j.Created.Before(cutoff) { // firmware jobs have their own watch
 			out = append(out, j)
 		}
 	}

cmd/rollcall/firmware.go

@@ -0,0 +1,868 @@
+package main
+
+// Firmware: a repository of vendor packages on the region, each machine's
+// firmware inventory read from its BMC, and update jobs.
+//
+//	<firmware_dir>/<id>/meta.json + the package file, served under /firmware/
+//
+// A package is for one vendor profile (the Redfish driver: idrac, ilo, ...),
+// one component (bios, bmc, nic, raid, cpld, psu, drive, other), a version,
+// and optionally models. An update is a job (kind "firmware") that the region
+// drives with short rack commands: fw-start hands the package to the BMC,
+// fw-poll watches the BMC's task until it ends, fw-inventory reads the
+// versions afterwards. The task is kept in the job, so a region restart
+// resumes watching instead of losing the update.
+
+import (
+	"archive/zip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+type FWPackage struct {
+	ID        string    `json:"id"`
+	Vendor    string    `json:"vendor"`    // a Redfish driver: idrac, ilo, xcc, supermicro, ...
+	Component string    `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other
+	Version   string    `json:"version"`
+	Models    []string  `json:"models,omitempty"` // model substrings it is for; none = any of the vendor's
+	File      string    `json:"file"`
+	Size      int64     `json:"size"`
+	SHA256    string    `json:"sha256"`
+	Source    string    `json:"source,omitempty"`
+	Notes     string    `json:"notes,omitempty"`
+	Imported  time.Time `json:"imported"`
+}
+
+var fwComponents = []string{"bios", "bmc", "nic", "raid", "cpld", "psu", "drive", "other"}
+
+var fwIDRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,80}$`)
+var fwFileRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,120}$`)
+
+const firmwareSchema = `
+CREATE TABLE IF NOT EXISTS firmware_inventory (
+	machine_id text PRIMARY KEY REFERENCES machines(id) ON DELETE CASCADE,
+	at         timestamptz NOT NULL DEFAULT now(),
+	components jsonb NOT NULL
+);
+`
+
+// ---------------------------------------------------------------- repository
+
+func fwList(dir string) ([]FWPackage, error) {
+	ents, err := os.ReadDir(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []FWPackage{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := []FWPackage{}
+	for _, e := range ents {
+		b, err := os.ReadFile(filepath.Join(dir, e.Name(), "meta.json"))
+		if err != nil {
+			continue
+		}
+		var p FWPackage
+		if json.Unmarshal(b, &p) == nil && p.ID == e.Name() {
+			out = append(out, p)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		a, b := out[i], out[j]
+		if a.Vendor != b.Vendor {
+			return a.Vendor < b.Vendor
+		}
+		if a.Component != b.Component {
+			return a.Component < b.Component
+		}
+		return a.Imported.After(b.Imported)
+	})
+	return out, nil
+}
+
+func fwGet(dir, id string) (FWPackage, error) {
+	if !fwIDRe.MatchString(id) {
+		return FWPackage{}, fmt.Errorf("no firmware package %q", id)
+	}
+	b, err := os.ReadFile(filepath.Join(dir, id, "meta.json"))
+	if err != nil {
+		return FWPackage{}, fmt.Errorf("no firmware package %q", id)
+	}
+	var p FWPackage
+	return p, json.Unmarshal(b, &p)
+}
+
+// fwCheck validates a package description against what Rollcall knows.
+func fwCheck(p *FWPackage) error {
+	p.Vendor, p.Component = strings.ToLower(p.Vendor), strings.ToLower(p.Component)
+	if d, ok := driverByName(p.Vendor); !ok || d.Family != "redfish" {
+		var names []string
+		for _, r := range redfishProfiles {
+			names = append(names, r.Driver)
+		}
+		return fmt.Errorf("vendor is a Redfish driver: %s", strings.Join(names, ", "))
+	}
+	if !hasTag(fwComponents, p.Component) {
+		return fmt.Errorf("component is one of %s", strings.Join(fwComponents, ", "))
+	}
+	if strings.TrimSpace(p.Version) == "" || strings.ContainsAny(p.Version, "\r\n") {
+		return errors.New("version is required")
+	}
+	if !fwFileRe.MatchString(p.File) {
+		return fmt.Errorf("file name %q: letters, digits and ._+- only", p.File)
+	}
+	if fs := redfishProfileFor(p.Vendor).FWFormats; len(fs) > 0 {
+		ok := false
+		for _, ext := range fs {
+			ok = ok || strings.HasSuffix(strings.ToLower(p.File), ext)
+		}
+		if !ok {
+			return fmt.Errorf("%s takes %s packages, not %s", p.Vendor, strings.Join(fs, " "), p.File)
+		}
+	}
+	return nil
+}
+
+// fwImport copies (or downloads) a package into the repository.
+func fwImport(ctx context.Context, dir, src string, p FWPackage, wantSum string) (*FWPackage, error) {
+	if p.File == "" {
+		p.File = path.Base(strings.SplitN(src, "?", 2)[0])
+	}
+	if err := fwCheck(&p); err != nil {
+		return nil, err
+	}
+	var in io.ReadCloser
+	if strings.HasPrefix(src, "https://") || strings.HasPrefix(src, "http://") {
+		req, err := http.NewRequestWithContext(ctx, "GET", src, nil)
+		if err != nil {
+			return nil, err
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode != 200 {
+			resp.Body.Close()
+			return nil, fmt.Errorf("%s: %s", src, resp.Status)
+		}
+		in = resp.Body
+	} else {
+		f, err := os.Open(src)
+		if err != nil {
+			return nil, err
+		}
+		in = f
+	}
+	defer in.Close()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, err
+	}
+	tmp, err := os.MkdirTemp(dir, ".import-")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmp)
+	f, err := os.Create(filepath.Join(tmp, p.File))
+	if err != nil {
+		return nil, err
+	}
+	h := sha256.New()
+	n, err := io.Copy(io.MultiWriter(f, h), in)
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return nil, err
+	}
+	p.Size, p.SHA256 = n, hex.EncodeToString(h.Sum(nil))
+	if wantSum != "" && !strings.EqualFold(wantSum, p.SHA256) {
+		return nil, fmt.Errorf("sha256 %s, expected %s", p.SHA256, wantSum)
+	}
+	if p.Vendor == "ilo" && strings.HasSuffix(strings.ToLower(p.File), ".fwpkg") {
+		if err := fwpkgExtract(tmp, &p); err != nil {
+			return nil, err
+		}
+	}
+	if p.Source == "" {
+		p.Source = src
+	}
+	p.Imported = time.Now().UTC()
+	slug := strings.Trim(regexp.MustCompile(`[^a-z0-9.]+`).ReplaceAllString(strings.ToLower(p.Vendor+"-"+p.Component+"-"+p.Version), "-"), "-")
+	if len(slug) > 60 {
+		slug = slug[:60]
+	}
+	p.ID = slug + "-" + p.SHA256[:8]
+	mb, _ := json.MarshalIndent(p, "", "  ")
+	if err := os.WriteFile(filepath.Join(tmp, "meta.json"), append(mb, '\n'), 0o644); err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(tmp, 0o755); err != nil {
+		return nil, err
+	}
+	dst := filepath.Join(dir, p.ID)
+	if _, err := os.Stat(dst); err == nil {
+		return &p, nil // the same package again
+	}
+	if err := os.Rename(tmp, dst); err != nil {
+		return nil, err
+	}
+	return &p, nil
+}
+
+// fwpkgExtract: iLO's SimpleUpdate flashes the iLO .bin or UEFI .signed.flash
+// inside an HPE Smart Component; given the .fwpkg it answers 200 and flashes
+// nothing. The .fwpkg is a zip: the one flashable file is kept instead.
+func fwpkgExtract(dir string, p *FWPackage) error {
+	src := filepath.Join(dir, p.File)
+	zr, err := zip.OpenReader(src)
+	if err != nil {
+		return fmt.Errorf("%s is not a readable .fwpkg: %w", p.File, err)
+	}
+	defer zr.Close()
+	var cands []*zip.File
+	for _, f := range zr.File {
+		n := strings.ToLower(path.Base(f.Name))
+		if !f.FileInfo().IsDir() && (strings.HasSuffix(n, ".bin") || strings.HasSuffix(n, ".flash")) && fwFileRe.MatchString(path.Base(f.Name)) {
+			cands = append(cands, f)
+		}
+	}
+	if len(cands) != 1 {
+		var names []string
+		for _, f := range cands {
+			names = append(names, f.Name)
+		}
+		return fmt.Errorf("%s holds %d files iLO flashes directly (%s); a component only UEFI or an OS agent installs needs SUM/iLOrest", p.File, len(cands), strings.Join(names, ", "))
+	}
+	in, err := cands[0].Open()
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	name := path.Base(cands[0].Name)
+	out, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		return err
+	}
+	h := sha256.New()
+	n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(in, 1<<30))
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return err
+	}
+	_ = os.Remove(src)
+	p.Notes = strings.TrimSpace(p.Notes + fmt.Sprintf(" extracted from %s (sha256 %s)", p.File, p.SHA256))
+	p.File, p.Size, p.SHA256 = name, n, hex.EncodeToString(h.Sum(nil))
+	return nil
+}
+
+func fwRemove(dir, id string) error {
+	if !fwIDRe.MatchString(id) {
+		return fmt.Errorf("no firmware package %q", id)
+	}
+	if _, err := os.Stat(filepath.Join(dir, id, "meta.json")); err != nil {
+		return fmt.Errorf("no firmware package %q", id)
+	}
+	return os.RemoveAll(filepath.Join(dir, id))
+}
+
+// fwFor: the packages for this machine (vendor, model), newest first per component.
+func fwFor(pkgs []FWPackage, vendor, model string) []FWPackage {
+	var out []FWPackage
+	for _, p := range pkgs {
+		if p.Vendor != vendor {
+			continue
+		}
+		if len(p.Models) > 0 {
+			ok := false
+			for _, m := range p.Models {
+				ok = ok || strings.Contains(strings.ToLower(model), strings.ToLower(m))
+			}
+			if !ok {
+				continue
+			}
+		}
+		out = append(out, p)
+	}
+	return out
+}
+
+// ---------------------------------------------------------------- inventory
+
+type fwInventory struct {
+	At         time.Time     `json:"at"`
+	Components []fwComponent `json:"components"`
+}
+
+func (s *Store) FirmwareInventory(ctx context.Context, id string) (*fwInventory, error) {
+	var inv fwInventory
+	var b []byte
+	err := s.pool.QueryRow(ctx, `SELECT at, components FROM firmware_inventory WHERE machine_id=$1`, id).Scan(&inv.At, &b)
+	if err != nil {
+		return nil, err
+	}
+	return &inv, json.Unmarshal(b, &inv.Components)
+}
+
+func (s *Store) PutFirmwareInventory(ctx context.Context, id string, cs []fwComponent) error {
+	b, _ := json.Marshal(cs)
+	_, err := s.pool.Exec(ctx, `INSERT INTO firmware_inventory(machine_id, at, components) VALUES ($1, now(), $2)
+		ON CONFLICT (machine_id) DO UPDATE SET at=now(), components=$2`, id, b)
+	return err
+}
+
+func (a *App) readFirmware(ctx context.Context, m *Machine) ([]fwComponent, error) {
+	out, err := a.firmwareVia(ctx, m, "fw-inventory", "")
+	if err != nil {
+		return nil, err
+	}
+	var cs []fwComponent
+	if err := json.Unmarshal([]byte(out), &cs); err != nil {
+		return nil, fmt.Errorf("inventory: %w", err)
+	}
+	return cs, a.store.PutFirmwareInventory(ctx, m.ID, cs)
+}
+
+// bmcVendor is the machine's Redfish driver and model, as probed.
+func bmcVendor(m *Machine) (driver, model string) {
+	var b bmcRecord
+	_ = json.Unmarshal(m.BMC, &b)
+	var rf struct {
+		Model string `json:"model"`
+	}
+	var raw obj
+	_ = json.Unmarshal(m.BMC, &raw)
+	if x, ok := raw["redfish"]; ok {
+		bb, _ := json.Marshal(x)
+		_ = json.Unmarshal(bb, &rf)
+	}
+	model = rf.Model
+	if model == "" {
+		model = deref(m.Product)
+	}
+	return b.Driver, model
+}
+
+// fwCompare marks, per component, whether a package in the repository is newer than what is installed.
+type fwOffer struct {
+	Package   FWPackage `json:"package"`
+	Installed string    `json:"installed"` // the version now on the component ("" when not seen)
+	Same      bool      `json:"same"`      // the package's version is what is installed
+}
+
+func fwOffers(pkgs []FWPackage, inv []fwComponent) []fwOffer {
+	var out []fwOffer
+	for _, p := range pkgs {
+		o := fwOffer{Package: p}
+		for _, c := range inv {
+			if c.Component == p.Component {
+				if o.Installed == "" {
+					o.Installed = c.Version
+				}
+				if fwSameVersion(c.Version, p.Version) {
+					o.Installed, o.Same = c.Version, true
+				}
+			}
+		}
+		out = append(out, o)
+	}
+	return out
+}
+
+// fwSameVersion compares loosely: "2.19.1" matches "2.19.1", "v2.19.1" and "2.19.1 (Build 4)".
+func fwSameVersion(installed, pkg string) bool {
+	norm := func(s string) string { return strings.TrimPrefix(strings.ToLower(strings.TrimSpace(s)), "v") }
+	i, p := norm(installed), norm(pkg)
+	return i == p || (p != "" && (strings.HasPrefix(i, p+" ") || strings.HasPrefix(i, p+"(") || strings.HasPrefix(i, p+"-")))
+}
+
+// ---------------------------------------------------------------- jobs
+
+type fwSpec struct {
+	Package   string `json:"package"`
+	Component string `json:"component"`
+	Version   string `json:"version"`
+	Apply     string `json:"apply,omitempty"`  // Immediate | OnReset
+	Method    string `json:"method,omitempty"` // forced upload method
+	Reboot    bool   `json:"reboot,omitempty"` // power-cycle the host when the update waits for a reset
+}
+
+type fwProgress struct {
+	Phase  string `json:"phase"` // uploading | flashing | rebooting | verifying
+	Pct    int    `json:"pct"`
+	Msg    string `json:"msg,omitempty"`
+	Method string `json:"method,omitempty"`
+	Task   string `json:"task,omitempty"`
+}
+
+func (a *App) setFWProgress(ctx context.Context, j *Job, p fwProgress) {
+	b, _ := json.Marshal(p)
+	_ = a.store.JobProgress(ctx, j.ID, b)
+}
+
+// firmwareURL is where the BMC downloads the package: the machine's rack
+// controller, or firmware_base on the BMC record when BMCs reach another address.
+func (a *App) firmwareURL(m *Machine, p FWPackage) string {
+	base := a.urlFor(m)
+	var b bmcRecord
+	if json.Unmarshal(m.BMC, &b) == nil && b.Extra["firmware_base"] != "" {
+		base = strings.TrimRight(b.Extra["firmware_base"], "/")
+	}
+	return base + "/firmware/" + p.ID + "/" + p.File
+}
+
+func (a *App) startFirmware(ctx context.Context, m *Machine, p FWPackage, sp fwSpec, force bool) (*Job, error) {
+	drv, model := bmcVendor(m)
+	if d, _ := driverByName(drv); d.Family != "redfish" {
+		return nil, fmt.Errorf("%s: firmware updates need a Redfish BMC driver (now %s): rollcall bmc %s -probe", m.ID, orDefault(drv, "ipmi"), m.ID)
+	}
+	if !force {
+		if p.Vendor != drv {
+			return nil, fmt.Errorf("%s is a %s package; this BMC is %s (-force to send it anyway)", p.ID, p.Vendor, drv)
+		}
+		if len(fwFor([]FWPackage{p}, drv, model)) == 0 {
+			return nil, fmt.Errorf("%s is for models %v, this is %q (-force to send it anyway)", p.ID, p.Models, model)
+		}
+	}
+	if j, err := a.store.ActiveJob(ctx, m.ID); err == nil {
+		return nil, fmt.Errorf("job %s (%s, %s) is still active", j.ID, j.Kind, j.State)
+	}
+	sp.Package, sp.Component, sp.Version = p.ID, p.Component, p.Version
+	specB, _ := json.Marshal(sp)
+	j := &Job{ID: "j-" + randomToken(5), MachineID: m.ID, Kind: "firmware", State: "pending", Spec: specB}
+	if err := a.store.CreateJob(ctx, j, "", nil); err != nil {
+		return nil, err
+	}
+	a.setFWProgress(ctx, j, fwProgress{Phase: "uploading", Msg: "handing " + p.File + " to the BMC"})
+	_ = a.store.AddEvent(ctx, m.ID, "firmware requested", fmt.Sprintf("job=%s %s %s %s apply=%s", j.ID, p.Component, p.Version, p.File, orDefault(sp.Apply, "default")))
+	go a.runFirmware(j.ID, m.ID, false)
+	return j, nil
+}
+
+// runFirmware drives one update to its end; resume picks up a task already started.
+func (a *App) runFirmware(jobID, machineID string, resume bool) {
+	ctx := context.Background()
+	j, err := a.store.Job(ctx, jobID)
+	if err != nil {
+		return
+	}
+	m, err := a.store.Get(ctx, machineID)
+	if err != nil {
+		return
+	}
+	var sp fwSpec
+	_ = json.Unmarshal(j.Spec, &sp)
+	var pr fwProgress
+	_ = json.Unmarshal(j.Progress, &pr)
+	finish := func(state, msg string) {
+		_ = a.store.FinishJob(ctx, j.ID, state, map[bool]string{true: "", false: msg}[state == "done"])
+		ev := "firmware " + state
+		_ = a.store.AddEvent(ctx, m.ID, ev, fmt.Sprintf("job=%s %s %s: %s", j.ID, sp.Component, sp.Version, msg))
+		log.Printf("%s: firmware %s %s %s: %s", m.ID, sp.Component, sp.Version, state, msg)
+	}
+	if !resume || pr.Task == "" && pr.Phase == "uploading" {
+		if resume { // the region went away while the upload was running: the BMC may or may not have it
+			finish("failed", "the region restarted during the upload; check the BMC, then try again")
+			return
+		}
+		p, err := fwGet(a.cfg.FirmwareDir, sp.Package)
+		if err != nil {
+			finish("failed", err.Error())
+			return
+		}
+		arg, _ := json.Marshal(fwStartArg{URL: a.firmwareURL(m, p), File: p.File, Component: p.Component, Method: sp.Method, Apply: sp.Apply})
+		out, err := a.firmwareVia(ctx, m, "fw-start", string(arg))
+		if err != nil {
+			finish("failed", err.Error())
+			return
+		}
+		var res fwStartResult
+		_ = json.Unmarshal([]byte(out), &res)
+		pr = fwProgress{Phase: "flashing", Method: res.Method, Task: res.Task, Msg: "the BMC has the package"}
+		a.setFWProgress(ctx, j, pr)
+	}
+	// watch the task; a BMC flashing itself drops off the network for minutes
+	if pr.Task != "" && pr.Phase == "flashing" {
+		var lastOK time.Time = time.Now()
+		for {
+			out, err := a.firmwareVia(ctx, m, "fw-poll", pr.Task)
+			if err != nil {
+				if time.Since(lastOK) > 25*time.Minute {
+					finish("failed", "the BMC has not answered for 25 minutes: "+err.Error())
+					return
+				}
+				pr.Msg = "BMC not answering (it may be restarting): " + oneLine(err.Error())
+				a.setFWProgress(ctx, j, pr)
+				time.Sleep(20 * time.Second)
+				continue
+			}
+			lastOK = time.Now()
+			var pl fwPollResult
+			_ = json.Unmarshal([]byte(out), &pl)
+			pr.Pct, pr.Msg = pl.Pct, pl.Msg
+			a.setFWProgress(ctx, j, pr)
+			if pl.State == "failed" {
+				_, _ = a.firmwareVia(ctx, m, "fw-finish", "")
+				finish("failed", orDefault(pl.Msg, "the BMC reported the update failed"))
+				return
+			}
+			if pl.State == "done" || pl.State == "unknown" {
+				break
+			}
+			if time.Since(*orNow(j.Started)) > 3*time.Hour {
+				finish("failed", "the BMC task did not end within 3 hours")
+				return
+			}
+			time.Sleep(15 * time.Second)
+		}
+	}
+	_, _ = a.firmwareVia(ctx, m, "fw-finish", "") // e.g. Lenovo's busy flag
+	// a component that applies at the next reset (BIOS, most NICs/RAID) waits for it
+	if sp.Reboot && pr.Phase != "verifying" {
+		pr.Phase, pr.Msg = "rebooting", "power cycle so the update applies"
+		a.setFWProgress(ctx, j, pr)
+		if _, err := a.power(ctx, m, "cycle", ""); err != nil {
+			finish("failed", "update staged, but the power cycle failed: "+err.Error())
+			return
+		}
+	}
+	pr.Phase, pr.Msg = "verifying", "reading the firmware inventory"
+	a.setFWProgress(ctx, j, pr)
+	deadline := time.Now().Add(20 * time.Minute) // servers take minutes to POST, BMCs to come back
+	for {
+		inv, err := a.readFirmware(ctx, m)
+		if err == nil {
+			for _, c := range inv {
+				if c.Component == sp.Component && fwSameVersion(c.Version, sp.Version) {
+					finish("done", "installed "+c.Name+" "+c.Version)
+					return
+				}
+			}
+		}
+		if time.Now().After(deadline) || (!sp.Reboot && err == nil && pr.Task != "") {
+			break
+		}
+		time.Sleep(60 * time.Second)
+	}
+	if sp.Reboot {
+		finish("failed", "the inventory does not show "+sp.Version+" after the update and a reboot")
+		return
+	}
+	// staged: most components switch at the next reset of the host
+	finish("done", "accepted by the BMC; "+sp.Version+" is not in the inventory yet (applies at the next reboot, or rerun with -reboot)")
+}
+
+func orNow(t *time.Time) *time.Time {
+	if t == nil {
+		n := time.Now()
+		return &n
+	}
+	return t
+}
+
+// resumeFirmware watches the updates that were running when the region stopped.
+func (a *App) resumeFirmware() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	active, err := a.store.ActiveJobs(ctx)
+	if err != nil {
+		return
+	}
+	for _, j := range active {
+		if j.Kind == "firmware" {
+			log.Printf("%s: resuming firmware job %s", j.MachineID, j.ID)
+			go a.runFirmware(j.ID, j.MachineID, true)
+		}
+	}
+}
+
+// ---------------------------------------------------------------- HTTP
+
+func (a *App) handleFirmwareRepo(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		pkgs, err := fwList(a.cfg.FirmwareDir)
+		if err != nil {
+			fail(w, 500, err)
+			return
+		}
+		writeJSON(w, 200, pkgs)
+	case http.MethodPost: // import from a URL: {"url", "vendor", "component", "version", "models", "sha256", "notes"}
+		var req struct {
+			FWPackage
+			URL    string `json:"url"`
+			SHA256 string `json:"sha256"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !(strings.HasPrefix(req.URL, "https://") || strings.HasPrefix(req.URL, "http://")) {
+			fail(w, 400, errors.New("url (http or https), vendor, component and version are required"))
+			return
+		}
+		ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute)
+		defer cancel()
+		p, err := fwImport(ctx, a.cfg.FirmwareDir, req.URL, req.FWPackage, req.SHA256)
+		if err != nil {
+			fail(w, 400, err)
+			return
+		}
+		log.Printf("firmware %s imported by %s from %s", p.ID, a.who(r), req.URL)
+		writeJSON(w, 200, p)
+	}
+}
+
+func (a *App) handleFirmwareRemove(w http.ResponseWriter, r *http.Request) {
+	if err := fwRemove(a.cfg.FirmwareDir, r.PathValue("pkg")); err != nil {
+		fail(w, 404, err)
+		return
+	}
+	writeJSON(w, 200, obj{"removed": r.PathValue("pkg")})
+}
+
+// handleFirmwareFile serves one package file (no listings): BMCs download from here.
+func (a *App) handleFirmwareFile(w http.ResponseWriter, r *http.Request) {
+	p, err := fwGet(a.cfg.FirmwareDir, r.PathValue("pkg"))
+	if err != nil || r.PathValue("file") != p.File {
+		http.NotFound(w, r)
+		return
+	}
+	http.ServeFile(w, r, filepath.Join(a.cfg.FirmwareDir, p.ID, p.File))
+}
+
+// handleMachineFirmware: GET the stored inventory and what the repository offers; POST ?refresh reads it again.
+func (a *App) handleMachineFirmware(w http.ResponseWriter, r *http.Request) {
+	m, err := a.store.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		fail(w, 404, err)
+		return
+	}
+	if r.Method == http.MethodPost {
+		ctx, cancel := context.WithTimeout(r.Context(), 6*time.Minute)
+		defer cancel()
+		if _, err := a.readFirmware(ctx, m); err != nil {
+			fail(w, 502, err)
+			return
+		}
+	}
+	inv, err := a.store.FirmwareInventory(r.Context(), m.ID)
+	if err != nil {
+		inv = &fwInventory{Components: []fwComponent{}}
+	}
+	drv, model := bmcVendor(m)
+	pkgs, _ := fwList(a.cfg.FirmwareDir)
+	offers := fwOffers(fwFor(pkgs, drv, model), inv.Components)
+	if offers == nil {
+		offers = []fwOffer{}
+	}
+	writeJSON(w, 200, obj{"id": m.ID, "driver": drv, "model": model, "inventory": inv, "offers": offers})
+}
+
+// handleFirmwareUpdate starts one update: {"package", "apply", "reboot", "method", "force"}.
+func (a *App) handleFirmwareUpdate(w http.ResponseWriter, r *http.Request) {
+	m, err := a.store.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		fail(w, 404, err)
+		return
+	}
+	var req struct {
+		fwSpec
+		Force bool `json:"force"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Package == "" {
+		fail(w, 400, errors.New("package is required"))
+		return
+	}
+	switch req.Apply {
+	case "", "Immediate", "OnReset":
+	default:
+		fail(w, 400, errors.New("apply is Immediate or OnReset"))
+		return
+	}
+	switch req.Method {
+	case "", "multipart", "simple", "push":
+	default:
+		fail(w, 400, errors.New("method is multipart, simple or push"))
+		return
+	}
+	p, err := fwGet(a.cfg.FirmwareDir, req.Package)
+	if err != nil {
+		fail(w, 404, err)
+		return
+	}
+	j, err := a.startFirmware(r.Context(), m, p, req.fwSpec, req.Force)
+	if err != nil {
+		fail(w, 409, err)
+		return
+	}
+	log.Printf("%s: firmware %s requested by %s (job %s)", m.ID, p.ID, a.who(r), j.ID)
+	writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "package": p.ID, "component": p.Component, "version": p.Version})
+}
+
+// ---------------------------------------------------------------- CLI
+
+func firmwareCmd(args []string) error {
+	if len(args) == 0 {
+		args = []string{"list"}
+	}
+	switch args[0] {
+	case "import":
+		if len(args) < 2 {
+			usage()
+		}
+		fs := newFlags("firmware import")
+		vendor := fs.String("vendor", "", "Redfish driver the package is for: idrac, ilo, xcc, supermicro, xfusion, kaytus, irmc, cimc, gigabyte, asrockrack, openbmc, ami")
+		comp := fs.String("component", "", strings.Join(fwComponents, " | "))
+		ver := fs.String("version", "", "the version the component reports once installed")
+		models := fs.String("models", "", "comma-separated model substrings it is for (default: any)")
+		sum := fs.String("sha256", "", "expected SHA-256 (from the vendor's download page)")
+		notes := fs.String("notes", "", "notes")
+		_ = fs.Parse(args[2:])
+		p := FWPackage{Vendor: *vendor, Component: *comp, Version: *ver, Notes: *notes}
+		if *models != "" {
+			p.Models = strings.Split(*models, ",")
+		}
+		src := args[1]
+		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { // a local file: on the region host
+			cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"))
+			if err != nil {
+				return fmt.Errorf("importing a file needs the region's config (run on the region, or give an https URL): %w", err)
+			}
+			got, err := fwImport(context.Background(), cfg.FirmwareDir, src, p, *sum)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256)
+			return nil
+		}
+		c, err := newClient()
+		if err != nil {
+			return err
+		}
+		var got FWPackage
+		req := obj{"url": src, "vendor": p.Vendor, "component": p.Component, "version": p.Version, "models": p.Models, "sha256": *sum, "notes": p.Notes}
+		if err := c.do("POST", "/api/v1/firmware", req, &got); err != nil {
+			return err
+		}
+		fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256)
+		return nil
+	}
+	c, err := newClient()
+	if err != nil {
+		return err
+	}
+	switch args[0] {
+	case "list":
+		var pkgs []FWPackage
+		if err := c.do("GET", "/api/v1/firmware", nil, &pkgs); err != nil {
+			return err
+		}
+		f := "%-44s %-11s %-9s %-22s %9s  %s\n"
+		fmt.Printf(f, "PACKAGE", "VENDOR", "COMPONENT", "VERSION", "SIZE", "FILE / MODELS")
+		for _, p := range pkgs {
+			ms := ""
+			if len(p.Models) > 0 {
+				ms = "  [" + strings.Join(p.Models, ", ") + "]"
+			}
+			fmt.Printf(f, p.ID, p.Vendor, p.Component, p.Version, human(float64(p.Size)), p.File+ms)
+		}
+	case "rm":
+		if len(args) < 2 {
+			usage()
+		}
+		return c.do("DELETE", "/api/v1/firmware/"+args[1], nil, nil)
+	case "show":
+		if len(args) < 2 {
+			usage()
+		}
+		fs := newFlags("firmware show")
+		refresh := fs.Bool("refresh", false, "read the inventory from the BMC now")
+		_ = fs.Parse(args[2:])
+		method := "GET"
+		if *refresh {
+			method = "POST"
+		}
+		var d struct {
+			Driver    string      `json:"driver"`
+			Model     string      `json:"model"`
+			Inventory fwInventory `json:"inventory"`
+			Offers    []fwOffer   `json:"offers"`
+		}
+		if err := c.do(method, "/api/v1/machines/"+args[1]+"/firmware", nil, &d); err != nil {
+			return err
+		}
+		fmt.Printf("%s  driver %s  model %s  inventory %s\n", args[1], orDash(d.Driver), orDash(d.Model), orDash(fwWhen(d.Inventory.At)))
+		if len(d.Inventory.Components) == 0 {
+			fmt.Println("no inventory yet: rollcall firmware show " + args[1] + " -refresh")
+		}
+		for _, x := range d.Inventory.Components {
+			up := ""
+			if !x.Updateable {
+				up = "  (not updateable)"
+			}
+			fmt.Printf("  %-9s %-40s %s%s\n", x.Component, trunc(x.Name, 40), x.Version, up)
+		}
+		if len(d.Offers) > 0 {
+			fmt.Println("repository:")
+			for _, o := range d.Offers {
+				st := "update"
+				if o.Same {
+					st = "installed"
+				}
+				fmt.Printf("  %-9s %-22s %-9s (now %s)  %s\n", o.Package.Component, o.Package.Version, st, orDash(o.Installed), o.Package.ID)
+			}
+		}
+	case "update":
+		if len(args) < 3 {
+			usage()
+		}
+		fs := newFlags("firmware update")
+		apply := fs.String("apply", "", "Immediate | OnReset (default: the BMC's)")
+		reboot := fs.Bool("reboot", false, "power-cycle the host afterwards so a staged update applies, then verify")
+		method := fs.String("method", "", "multipart | simple | push (default: the vendor profile's order)")
+		force := fs.Bool("force", false, "send a package for another vendor or model")
+		watch := fs.Bool("watch", false, "follow the job")
+		_ = fs.Parse(args[3:])
+		ids, err := c.targets(args[1])
+		if err != nil {
+			return err
+		}
+		req := obj{"package": args[2], "apply": *apply, "reboot": *reboot, "method": *method, "force": *force}
+		for _, id := range ids {
+			var d obj
+			if err := c.do("POST", "/api/v1/machines/"+id+"/firmware/update", req, &d); err != nil {
+				if len(ids) == 1 {
+					return err
+				}
+				fmt.Printf("%s: error: %v\n", id, err)
+				continue
+			}
+			fmt.Printf("%s: job %s  %s %s\n", id, str(d["job"]), str(d["component"]), str(d["version"]))
+		}
+		if *watch && len(ids) == 1 {
+			return c.watchJob(ids[0])
+		}
+	default:
+		usage()
+	}
+	return nil
+}
+
+func fwWhen(t time.Time) string {
+	if t.IsZero() {
+		return ""
+	}
+	return t.Local().Format("2006-01-02 15:04")
+}

cmd/rollcall/firmware_test.go

@@ -0,0 +1,289 @@
+package main
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/json"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// fwBMC is an UpdateService: it takes packages the ways the flags allow and
+// keeps a task per upload.
+type fwBMC struct {
+	mu                      sync.Mutex
+	multipart, simple, push bool
+	protocols               []string
+	got                     []string // method:file-content or method:ImageURI
+	params                  []obj
+	chunked                 bool
+	polls                   int
+}
+
+func (f *fwBMC) handler(t *testing.T) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if u, p, ok := r.BasicAuth(); !ok || u != "admin" || p != "secret" {
+			w.WriteHeader(401)
+			return
+		}
+		task := func() {
+			w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/3")
+			w.WriteHeader(202)
+		}
+		switch {
+		case r.URL.Path == "/redfish/v1/UpdateService":
+			us := obj{}
+			if f.multipart {
+				us["MultipartHttpPushUri"] = "/redfish/v1/UpdateService/upload"
+			}
+			if f.push {
+				us["HttpPushUri"] = "/redfish/v1/UpdateService/push"
+			}
+			if f.simple {
+				act := obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}
+				if f.protocols != nil {
+					act["TransferProtocol@Redfish.AllowableValues"] = f.protocols
+				}
+				us["Actions"] = obj{"#UpdateService.SimpleUpdate": act}
+			}
+			_ = json.NewEncoder(w).Encode(us)
+		case r.URL.Path == "/redfish/v1/UpdateService/upload":
+			f.chunked = f.chunked || r.ContentLength < 0 || len(r.TransferEncoding) > 0
+			_, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+			if err != nil {
+				w.WriteHeader(400)
+				return
+			}
+			mr := multipart.NewReader(r.Body, ps["boundary"])
+			for {
+				p, err := mr.NextPart()
+				if err != nil {
+					break
+				}
+				b, _ := io.ReadAll(p)
+				switch p.FormName() {
+				case "UpdateParameters":
+					var o obj
+					_ = json.Unmarshal(b, &o)
+					f.params = append(f.params, o)
+				case "UpdateFile":
+					f.got = append(f.got, "multipart:"+p.FileName()+":"+string(b))
+				}
+			}
+			task()
+		case r.URL.Path == "/redfish/v1/UpdateService/push":
+			f.chunked = f.chunked || r.ContentLength < 0
+			b, _ := io.ReadAll(r.Body)
+			f.got = append(f.got, "push:"+string(b))
+			task()
+		case r.URL.Path == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate":
+			var o obj
+			_ = json.NewDecoder(r.Body).Decode(&o)
+			f.got = append(f.got, "simple:"+str(o["ImageURI"])+":"+str(o["TransferProtocol"]))
+			task()
+		case r.URL.Path == "/redfish/v1/TaskService/Tasks/3":
+			f.polls++
+			st := obj{"TaskState": "Running", "PercentComplete": 40}
+			if f.polls > 1 {
+				st = obj{"TaskState": "Completed", "PercentComplete": 100, "Messages": []obj{{"Message": "Firmware update completed"}}}
+			}
+			_ = json.NewEncoder(w).Encode(st)
+		case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory":
+			_ = json.NewEncoder(w).Encode(obj{"Members": []obj{
+				{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-159-2.19.1"},
+				{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Previous-159-2.18.0"},
+				{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227-7.10.30.00"},
+			}})
+		case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-159"):
+			_ = json.NewEncoder(w).Encode(obj{"Id": "Installed-159-2.19.1", "Name": "BIOS", "Version": "2.19.1", "Updateable": true})
+		case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227"):
+			_ = json.NewEncoder(w).Encode(obj{"Id": "Installed-25227-7.10.30.00", "Name": "Integrated Dell Remote Access Controller", "Version": "7.10.30.00", "Updateable": true})
+		default:
+			http.NotFound(w, r)
+		}
+	})
+}
+
+func runFW(t *testing.T, f *fwBMC, op string, arg any) string {
+	t.Helper()
+	srv := httptest.NewTLSServer(f.handler(t))
+	defer srv.Close()
+	pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("FIRMWARE-BYTES")) }))
+	defer pkg.Close()
+	a := ""
+	if s, ok := arg.(string); ok {
+		a = s
+	} else if arg != nil {
+		sa := arg.(fwStartArg)
+		if sa.URL == "" {
+			sa.URL = pkg.URL + "/firmware/p/BIOS_X.EXE"
+		}
+		b, _ := json.Marshal(sa)
+		a = string(b)
+	}
+	out, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), op, a)
+	if err != nil {
+		t.Fatalf("%s: %v", op, err)
+	}
+	return out
+}
+
+func TestFirmwareMultipart(t *testing.T) {
+	f := &fwBMC{multipart: true, simple: true}
+	var res fwStartResult
+	_ = json.Unmarshal([]byte(runFW(t, f, "fw-start", fwStartArg{File: "BIOS_X.EXE", Apply: "OnReset"})), &res)
+	if res.Method != "multipart" || res.Task != "/redfish/v1/TaskService/Tasks/3" {
+		t.Errorf("result %+v", res)
+	}
+	if len(f.got) != 1 || f.got[0] != "multipart:BIOS_X.EXE:FIRMWARE-BYTES" || f.chunked {
+		t.Errorf("got %v chunked %v", f.got, f.chunked)
+	}
+	if f.params[0]["@Redfish.OperationApplyTime"] != "OnReset" {
+		t.Errorf("params %v", f.params)
+	}
+}
+
+func TestFirmwareSimpleAndPush(t *testing.T) {
+	f := &fwBMC{simple: true, protocols: []string{"HTTP", "HTTPS", "NFS"}}
+	_ = runFW(t, f, "fw-start", fwStartArg{File: "x.bin"})
+	if len(f.got) != 1 || !strings.HasPrefix(f.got[0], "simple:http://") || !strings.HasSuffix(f.got[0], ":HTTP") {
+		t.Errorf("simple %v", f.got)
+	}
+	g := &fwBMC{push: true}
+	_ = runFW(t, g, "fw-start", fwStartArg{File: "x.bin"})
+	if len(g.got) != 1 || g.got[0] != "push:FIRMWARE-BYTES" || g.chunked {
+		t.Errorf("push %v chunked %v", g.got, g.chunked)
+	}
+	// a service that pulls only over NFS cannot take our HTTP URL: refused clearly
+	h := &fwBMC{simple: true, protocols: []string{"NFS"}}
+	srv := httptest.NewTLSServer(h.handler(t))
+	defer srv.Close()
+	b, _ := json.Marshal(fwStartArg{URL: "http://rack/firmware/p/x.bin", File: "x.bin"})
+	if _, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), "fw-start", string(b)); err == nil || !strings.Contains(err.Error(), "not HTTP") {
+		t.Errorf("nfs-only: %v", err)
+	}
+}
+
+func TestFirmwarePollAndInventory(t *testing.T) {
+	f := &fwBMC{}
+	var p fwPollResult
+	_ = json.Unmarshal([]byte(runFW(t, f, "fw-poll", "/redfish/v1/TaskService/Tasks/3")), &p)
+	if p.State != "running" || p.Pct != 40 {
+		t.Errorf("first poll %+v", p)
+	}
+	var inv []fwComponent
+	_ = json.Unmarshal([]byte(runFW(t, f, "fw-inventory", nil)), &inv)
+	if len(inv) != 2 || inv[0].Component != "bios" || inv[1].Component != "bmc" || inv[0].Version != "2.19.1" {
+		t.Errorf("inventory %+v", inv)
+	}
+	pkgs := []FWPackage{{ID: "a", Vendor: "idrac", Component: "bios", Version: "2.19.1"}, {ID: "b", Vendor: "idrac", Component: "bmc", Version: "7.20.10.50"}}
+	offers := fwOffers(pkgs, inv)
+	if !offers[0].Same || offers[1].Same || offers[1].Installed != "7.10.30.00" {
+		t.Errorf("offers %+v", offers)
+	}
+}
+
+func TestFirmwareNeedsRedfish(t *testing.T) {
+	if _, err := powerRun(context.Background(), "", &bmcCreds{Driver: "ipmi"}, "fw-inventory", ""); err == nil || !strings.Contains(err.Error(), "Redfish") {
+		t.Errorf("ipmi: %v", err)
+	}
+}
+
+func TestFirmwareRepository(t *testing.T) {
+	dir := t.TempDir()
+	src := filepath.Join(t.TempDir(), "BIOS_ABC_WN64_2.19.1.EXE")
+	_ = os.WriteFile(src, []byte("dup"), 0o644)
+	p, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "2.19.1", Models: []string{"R650"}}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !fwIDRe.MatchString(p.ID) || p.Size != 3 || p.File != "BIOS_ABC_WN64_2.19.1.EXE" {
+		t.Errorf("package %+v", p)
+	}
+	if _, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "x"}, "00"); err == nil {
+		t.Error("a wrong sha256 was accepted")
+	}
+	for _, bad := range []FWPackage{{Vendor: "ipmi", Component: "bios", Version: "1"}, {Vendor: "idrac", Component: "toaster", Version: "1"}, {Vendor: "idrac", Component: "bios"}} {
+		if _, err := fwImport(context.Background(), dir, src, bad, ""); err == nil {
+			t.Errorf("accepted %+v", bad)
+		}
+	}
+	list, _ := fwList(dir)
+	if len(list) != 1 || len(fwFor(list, "idrac", "PowerEdge R650")) != 1 || len(fwFor(list, "idrac", "PowerEdge R760")) != 0 || len(fwFor(list, "ilo", "R650")) != 0 {
+		t.Errorf("list %+v", list)
+	}
+	if _, err := fwGet(dir, "../etc"); err == nil {
+		t.Error("a path as package id")
+	}
+	if err := fwRemove(dir, p.ID); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range [][2]string{{"2.19.1", "2.19.1"}, {"v2.19.1", "2.19.1"}, {"2.19.1 (Build 4)", "2.19.1"}} {
+		if !fwSameVersion(c[0], c[1]) {
+			t.Errorf("%q vs %q", c[0], c[1])
+		}
+	}
+	if fwSameVersion("2.19.10", "2.19.1") {
+		t.Error("2.19.10 is not 2.19.1")
+	}
+}
+
+func TestFirmwareKinds(t *testing.T) {
+	for in, want := range map[string]string{
+		"Installed-159-2.19.1 BIOS":                "bios",
+		"Integrated Dell Remote Access Controller": "bmc",
+		"PERC H755 Front":                          "raid",
+		"RAID Controller":                          "raid",
+		"iLO 6":                                    "bmc",
+		"Broadcom Adv. Dual 25Gb Ethernet":         "nic",
+		"Mellanox ConnectX-6":                      "nic",
+		"PSU.Slot.1 Power Supply":                  "psu",
+		"System CPLD":                              "cpld",
+		"Dell Ent NVMe P5600":                      "drive",
+		"Lifecycle Controller":                     "other",
+		"OS Collector":                             "other",
+	} {
+		if got := fwKind("", in); got != want {
+			t.Errorf("%q: %s, want %s", in, got, want)
+		}
+	}
+}
+
+func TestFirmwareFwpkg(t *testing.T) {
+	mk := func(files ...string) string {
+		p := filepath.Join(t.TempDir(), "cp056789.fwpkg")
+		f, _ := os.Create(p)
+		zw := zip.NewWriter(f)
+		for _, n := range files {
+			w, _ := zw.Create(n)
+			_, _ = w.Write([]byte("payload of " + n))
+		}
+		_ = zw.Close()
+		_ = f.Close()
+		return p
+	}
+	dir := t.TempDir()
+	got, err := fwImport(context.Background(), dir, mk("payload.json", "ilo6_166.bin"), FWPackage{Vendor: "ilo", Component: "bmc", Version: "1.66"}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.File != "ilo6_166.bin" || got.Size != int64(len("payload of ilo6_166.bin")) || !strings.Contains(got.Notes, "cp056789.fwpkg") {
+		t.Errorf("extracted %+v", got)
+	}
+	if _, err := os.Stat(filepath.Join(dir, got.ID, "ilo6_166.bin")); err != nil {
+		t.Error(err)
+	}
+	if _, err := fwImport(context.Background(), dir, mk("payload.json", "setup.exe"), FWPackage{Vendor: "ilo", Component: "nic", Version: "1"}, ""); err == nil || !strings.Contains(err.Error(), "0 files") {
+		t.Errorf("agent-only component: %v", err)
+	}
+}

cmd/rollcall/fwplans_test.go

@@ -0,0 +1,306 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// recBMC records what a firmware update sends: every request, the multipart
+// parts (JSON parsed), plain form fields and the file's field name.
+type recBMC struct {
+	mu        sync.Mutex
+	vendor    string
+	us        obj      // the UpdateService resource
+	inventory []string // FirmwareInventory ids
+	missing   []string // paths that answer 404
+	reqs      []recReq
+}
+
+type recReq struct {
+	method, path, ifMatch string
+	json                  obj            // JSON body, or the UpdateParameters part
+	parts                 map[string]obj // JSON parts by name
+	fields                map[string]string
+	fileField, fileName   string
+	fileBody              string
+}
+
+func (f *recBMC) find(method, pathPrefix string) *recReq {
+	for i := range f.reqs {
+		if f.reqs[i].method == method && strings.HasPrefix(f.reqs[i].path, pathPrefix) {
+			return &f.reqs[i]
+		}
+	}
+	return nil
+}
+
+func (f *recBMC) handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		for _, m := range f.missing {
+			if r.URL.Path == m {
+				http.NotFound(w, r)
+				return
+			}
+		}
+		if r.URL.Path == "/redfish/v1/SessionService/Sessions" && r.Method == "POST" {
+			w.Header().Set("X-Auth-Token", "tok")
+			w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/1")
+			w.WriteHeader(201)
+			return
+		}
+		if r.Method == "DELETE" {
+			w.WriteHeader(204)
+			return
+		}
+		rq := recReq{method: r.Method, path: r.URL.Path, ifMatch: r.Header.Get("If-Match"), parts: map[string]obj{}, fields: map[string]string{}}
+		if mt, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type")); err == nil && mt == "multipart/form-data" {
+			mr := multipart.NewReader(r.Body, ps["boundary"])
+			for {
+				p, err := mr.NextPart()
+				if err != nil {
+					break
+				}
+				b, _ := io.ReadAll(p)
+				switch {
+				case p.FileName() != "":
+					rq.fileField, rq.fileName, rq.fileBody = p.FormName(), p.FileName(), string(b)
+				case strings.HasPrefix(p.Header.Get("Content-Type"), "application/json"):
+					var o obj
+					_ = json.Unmarshal(b, &o)
+					rq.parts[p.FormName()] = o
+				default:
+					rq.fields[p.FormName()] = string(b)
+				}
+			}
+			rq.json = rq.parts["UpdateParameters"]
+		} else if r.Body != nil {
+			_ = json.NewDecoder(r.Body).Decode(&rq.json)
+		}
+		if r.Method != "GET" {
+			f.reqs = append(f.reqs, rq)
+		}
+		switch {
+		case r.URL.Path == "/redfish/v1/":
+			_ = json.NewEncoder(w).Encode(obj{"Vendor": f.vendor})
+		case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "GET":
+			w.Header().Set("ETag", `"us1"`)
+			_ = json.NewEncoder(w).Encode(f.us)
+		case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "PATCH":
+			for k, v := range rq.json {
+				f.us[k] = v
+			}
+			w.WriteHeader(204)
+		case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory" && r.Method == "GET":
+			var ms []obj
+			for _, id := range f.inventory {
+				ms = append(ms, obj{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/" + id})
+			}
+			_ = json.NewEncoder(w).Encode(obj{"Members": ms})
+		case strings.HasPrefix(r.URL.Path, "/redfish/v1/TaskService/Tasks/"):
+			_ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100})
+		case r.Method == "POST":
+			w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/9")
+			w.WriteHeader(202)
+		default:
+			http.NotFound(w, r)
+		}
+	})
+}
+
+// fwStartWith runs fw-start for a vendor driver and returns the result and the recorder.
+func fwStartWith(t *testing.T, f *recBMC, driver string, a fwStartArg, extra map[string]string) (*fwStartResult, error) {
+	t.Helper()
+	srv := httptest.NewTLSServer(f.handler())
+	defer srv.Close()
+	pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("IMG")) }))
+	defer pkg.Close()
+	if a.URL == "" {
+		a.URL = pkg.URL + "/firmware/p/" + a.File
+	}
+	c := creds(srv.URL, driver)
+	for k, v := range extra {
+		c.Extra[k] = v
+	}
+	b, _ := json.Marshal(a)
+	out, err := powerRun(context.Background(), "", c, "fw-start", string(b))
+	if err != nil {
+		return nil, err
+	}
+	var res fwStartResult
+	_ = json.Unmarshal([]byte(out), &res)
+	return &res, nil
+}
+
+func mustStart(t *testing.T, f *recBMC, driver string, a fwStartArg) *fwStartResult {
+	t.Helper()
+	res, err := fwStartWith(t, f, driver, a, nil)
+	if err != nil {
+		t.Fatalf("%s: %v", driver, err)
+	}
+	return res
+}
+
+func TestFWPlanDell(t *testing.T) {
+	f := &recBMC{vendor: "Dell", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/MultipartUpload"}}
+	mustStart(t, f, "idrac", fwStartArg{File: "BIOS_X_WN64_2.19.1.EXE", Component: "bios"})
+	rq := f.find("POST", "/redfish/v1/UpdateService/MultipartUpload")
+	if rq == nil || rq.json["@Redfish.OperationApplyTime"] != "OnReset" || rq.fileField != "UpdateFile" || rq.fileBody != "IMG" {
+		t.Errorf("dell %+v", rq)
+	}
+}
+
+func TestFWPlanSupermicro(t *testing.T) {
+	f := &recBMC{vendor: "Supermicro", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BIOS", "Motherboard_CPLD_1"}}
+	mustStart(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"})
+	rq := f.find("POST", "/redfish/v1/UpdateService/upload")
+	if rq == nil || dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveNVRAM") != true || toList(rq.json["Targets"])[0] != "/redfish/v1/Systems/1/Bios" {
+		t.Errorf("bios params %v", rq.json)
+	}
+	f.reqs = nil
+	mustStart(t, f, "supermicro", fwStartArg{File: "CPLD.bin", Component: "cpld"})
+	if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/Motherboard_CPLD_1" {
+		t.Errorf("cpld params %v", rq.json)
+	}
+	// the operator's own parameters (Gen 13 preserve flags)
+	f.reqs = nil
+	if _, err := fwStartWith(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"}, map[string]string{"fw_params": `{"Targets":["/redfish/v1/Systems/1/Bios"],"Oem":{"Supermicro":{"BIOS":{"PreserveOA":true}}}}`}); err != nil {
+		t.Fatal(err)
+	}
+	if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveOA") != true {
+		t.Errorf("fw_params %v", rq.json)
+	}
+}
+
+func TestFWPlanAMI(t *testing.T) {
+	f := &recBMC{us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BMCImage1", "BIOS", "BIOS2", "MB_CPLD1"}}
+	mustStart(t, f, "gigabyte", fwStartArg{File: "rom.ima_enc", Component: "bmc"})
+	rq := f.find("POST", "/redfish/v1/UpdateService/upload")
+	if rq.parts["OemParameters"]["ImageType"] != "BMC" || toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/BMCImage1" {
+		t.Errorf("gigabyte bmc %+v", rq)
+	}
+	f.reqs = nil
+	mustStart(t, f, "gigabyte", fwStartArg{File: "bios.hpm", Component: "bios"})
+	if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); rq.parts["OemParameters"]["ImageType"] != "HPM_BIOS" {
+		t.Errorf("gigabyte hpm bios %+v", rq.parts)
+	}
+	if _, err := fwStartWith(t, &recBMC{us: obj{}, inventory: []string{"BMC"}}, "ami", fwStartArg{File: "x.cap", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "no firmware BIOS") {
+		t.Errorf("missing target: %v", err)
+	}
+}
+
+func TestFWPlanXFusion(t *testing.T) {
+	f := &recBMC{us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
+	mustStart(t, f, "xfusion", fwStartArg{File: "image.hpm", Component: "bmc"})
+	up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate")
+	if up == nil || up.fileField != "file" || len(up.parts) != 0 || su == nil || su.json["ImageURI"] != "/tmp/web/image.hpm" {
+		t.Errorf("upload %+v simple %+v", up, su)
+	}
+	if _, err := fwStartWith(t, f, "xfusion", fwStartArg{File: "nic.hpm", Component: "nic"}, nil); err == nil {
+		t.Error("in-band NIC firmware accepted")
+	}
+}
+
+func TestFWPlanKaytus(t *testing.T) {
+	f := &recBMC{us: obj{"ServiceEnabled": true, "Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
+	res := mustStart(t, f, "kaytus", fwStartArg{File: "bios.bin", Component: "bios"})
+	up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions")
+	if up.fileField != "fwimage" || up.fields["CurrentPassword"] != "secret" || up.fields["EncryptFlag"] != "False" {
+		t.Errorf("upload %+v", up)
+	}
+	if dig(su.json, "Oem", "Public", "FlashItem") != "BIOS" || dig(su.json, "Oem", "Public", "BiosFlash") != "Both" {
+		t.Errorf("flash %v", su.json)
+	}
+	if res.Task == "" {
+		t.Errorf("no task")
+	}
+}
+
+func TestFWPlanIRMC(t *testing.T) {
+	f := &recBMC{vendor: "ts_fujitsu", us: obj{}, missing: []string{"/redfish/v1/Managers/iRMC/Actions/FTSManager.FWUpdate"}}
+	mustStart(t, f, "irmc", fwStartArg{File: "D3279_09.09F.bin", Component: "bmc"})
+	rq := f.find("POST", "/redfish/v1/Managers/iRMC/Actions/Oem/FTSManager.FWUpdate")
+	if rq == nil || rq.fileField != "data" || rq.json != nil {
+		t.Errorf("irmc %+v (all %+v)", rq, f.reqs)
+	}
+	g := &recBMC{vendor: "Fsas", us: obj{}}
+	mustStart(t, g, "irmc", fwStartArg{File: "D3279-B1x.R1.20.0.UPC", Component: "bios"})
+	if g.find("POST", "/redfish/v1/Systems/0/Bios/Actions/Oem/FsasBios.BiosUpdate") == nil {
+		t.Errorf("fsas bios %+v", g.reqs)
+	}
+}
+
+func TestFWPlanLenovoClaims(t *testing.T) {
+	f := &recBMC{vendor: "Lenovo", us: obj{"MultipartHttpPushUri": "/mfwupdate", "HttpPushUriTargetsBusy": false}}
+	mustStart(t, f, "xcc", fwStartArg{File: "lnvgy_fw_xcc.uxz", Component: "bmc"})
+	if f.us["HttpPushUriTargetsBusy"] != true || f.find("POST", "/mfwupdate") == nil || f.find("PATCH", "/redfish/v1/UpdateService").ifMatch != `"us1"` {
+		t.Errorf("claim %v %+v", f.us, f.reqs)
+	}
+	if _, err := fwStartWith(t, f, "xcc", fwStartArg{File: "x.uxz", Component: "bmc"}, nil); err == nil || !strings.Contains(err.Error(), "busy") {
+		t.Errorf("second upload while busy: %v", err)
+	}
+	srv := httptest.NewTLSServer(f.handler())
+	defer srv.Close()
+	if _, err := powerRun(context.Background(), "", creds(srv.URL, "xcc"), "fw-finish", ""); err != nil || f.us["HttpPushUriTargetsBusy"] != false {
+		t.Errorf("release: %v %v", err, f.us)
+	}
+}
+
+func TestFWPlanILOAndCisco(t *testing.T) {
+	f := &recBMC{vendor: "HPE", us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
+	mustStart(t, f, "ilo", fwStartArg{File: "ilo6_166.bin", Component: "bmc"})
+	if rq := f.find("POST", "/redfish/v1/UpdateService/Actions"); rq == nil || !strings.HasSuffix(str(rq.json["ImageURI"]), "/ilo6_166.bin") {
+		t.Errorf("ilo %+v", f.reqs)
+	}
+	if _, err := fwStartWith(t, &recBMC{us: obj{}}, "cimc", fwStartArg{File: "x.bin", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "SCP") {
+		t.Errorf("cimc: %v", err)
+	}
+}
+
+func TestFWPollStaged(t *testing.T) {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/redfish/v1/TaskService/Tasks/JID_1":
+			_ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 100, "Messages": []obj{{"Message": "Job scheduled successfully."}}})
+		case "/redfish/v1/TaskService/Tasks/BIOS0":
+			_ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100})
+		case "/redfish/v1/TaskService/Tasks/BIOS1":
+			_ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 30})
+		case "/redfish/v1/JobService/Jobs/JID_2":
+			_ = json.NewEncoder(w).Encode(obj{"JobState": "Completed", "PercentComplete": 100})
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	poll := func(task string) fwPollResult {
+		out, err := powerRun(context.Background(), "", creds(srv.URL, "idrac"), "fw-poll", task)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var p fwPollResult
+		_ = json.Unmarshal([]byte(out), &p)
+		return p
+	}
+	if p := poll("/redfish/v1/TaskService/Tasks/JID_1"); p.State != "done" {
+		t.Errorf("scheduled job: %+v", p)
+	}
+	if p := poll("/redfish/v1/TaskService/Tasks/BIOS0,/redfish/v1/TaskService/Tasks/BIOS1"); p.State != "running" || p.Pct != 30 {
+		t.Errorf("second of two: %+v", p)
+	}
+	if p := poll("/redfish/v1/TaskService/Tasks/JID_2"); p.State != "done" { // moved to the job service
+		t.Errorf("job service: %+v", p)
+	}
+	if p := poll("/redfish/v1/TaskService/Tasks/GONE"); p.State != "unknown" {
+		t.Errorf("gone: %+v", p)
+	}
+}

cmd/rollcall/ipmi.go

@@ -11,6 +11,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"
 )
 
 // bmcCreds is what a power driver needs.
@@ -83,6 +84,24 @@ func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg st
 		Password: c.Password, Extra: c.Extra, Op: op, Arg: arg})
 }
 
+// firmwareVia runs a firmware operation (fw-*) where the BMC is reachable.
+func (a *App) firmwareVia(ctx context.Context, m *Machine, op, arg string) (string, error) {
+	c, err := a.credsFor(m)
+	if err != nil {
+		return "", fmt.Errorf("%w: %v", errNoBMC, err)
+	}
+	rack := a.rackOf(m)
+	if rack == a.cfg.RackName || rack == "" {
+		return powerRun(ctx, a.cfg.IPMITool, c, op, arg)
+	}
+	r, ok := a.racksSnapshot()[rack]
+	if !ok || !rackOnline(r, a.cfg.RackName) {
+		return "", fmt.Errorf("rack controller %s is offline", rack)
+	}
+	return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User,
+		Password: c.Password, Extra: c.Extra, Op: op, Arg: arg, Timeout: int((fwTimeouts[op] + time.Minute).Seconds())})
+}
+
 func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error {
 	if secret != nil {
 		_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret)

cmd/rollcall/main.go

@@ -49,6 +49,8 @@ func usage() {
   rollcall unassign ID
   rollcall overlay ID [-stage system|runtime] [-extract DIR]
   rollcall render PROFILE PATH [-stage system|runtime]
+  rollcall firmware list | import FILE|URL -vendor V -component C -version X [-models M,..] [-sha256 S] | rm PKG
+  rollcall firmware show MACHINE [-refresh] | update MACHINE|tag:T PKG [-apply OnReset|Immediate] [-reboot] [-method M] [-force] [-watch]
   rollcall os list | sync [NAME...] [-force] | import NAME FILE [-family F] [-version V] | promote NAME VERSION | rm NAME VERSION
   rollcall deploy ID OS [-version V] [-disk /dev/sdX] [-hostname H] [-upgrade] [-user-data FILE] [-boot] [-watch]
   rollcall release ID [-boot] [-watch]
@@ -147,6 +149,11 @@ func main() {
 			fmt.Fprintln(os.Stderr, "error:", err)
 			os.Exit(1)
 		}
+	case "firmware":
+		if err := firmwareCmd(args); err != nil {
+			fmt.Fprintln(os.Stderr, "error:", err)
+			os.Exit(1)
+		}
 	case "os":
 		if err := osCmd(args); err != nil {
 			fmt.Fprintln(os.Stderr, "error:", err)

cmd/rollcall/power.go

@@ -57,6 +57,11 @@ var errManualPower = errors.New("this machine has manual power control: do it by
 
 // powerRun executes one operation with a driver.
 func powerRun(ctx context.Context, ipmitool string, c *bmcCreds, op, arg string) (string, error) {
+	if t, ok := fwTimeouts[op]; ok { // firmware: Redfish UpdateService (redfish_fw.go)
+		ctx, cancel := context.WithTimeout(ctx, t)
+		defer cancel()
+		return firmwareRun(ctx, c, op, arg)
+	}
 	if !powerOps[op] {
 		return "", fmt.Errorf("unknown power operation %q", op)
 	}

cmd/rollcall/rackctl.go

@@ -76,6 +76,7 @@ type rackJob struct {
 	Extra    map[string]string `json:"extra"`
 	Op       string            `json:"op"`
 	Arg      string            `json:"arg"`
+	Timeout  int               `json:"timeout,omitempty"` // seconds the region waits (default 90)
 	done     chan rackResult
 	rack     string // who took it: only that rack may answer
 }
@@ -242,7 +243,11 @@ func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string
 		delete(h.wake, rack)
 	}
 	h.mu.Unlock()
-	t := time.NewTimer(90 * time.Second)
+	wait := 90 * time.Second
+	if j.Timeout > 0 {
+		wait = time.Duration(j.Timeout) * time.Second
+	}
+	t := time.NewTimer(wait)
 	defer t.Stop()
 	select {
 	case r := <-j.done:

cmd/rollcall/rackd.go

@@ -358,7 +358,7 @@ func runRack(args []string) error {
 	}
 	go r.loop()
 	mux := http.NewServeMux()
-	for _, p := range []string{"/rollcall/", "/images/", "/os/"} {
+	for _, p := range []string{"/rollcall/", "/images/", "/os/", "/firmware/"} { // firmware: BMCs pull packages from here
 		mux.HandleFunc("GET "+p, r.serveCached)
 		mux.HandleFunc("HEAD "+p, r.serveCached)
 	}
@@ -546,8 +546,8 @@ func (r *rackd) head(rel string) (headInfo, error) {
 // fresh says whether the cached copy matches the region's file. OS images live
 // under versioned paths and never change; the rest is checked by size and time.
 func (r *rackd) fresh(rel string, fi os.FileInfo) bool {
-	if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") {
-		return true
+	if (strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/")) || strings.HasPrefix(rel, "/firmware/") {
+		return true // packages live under their id and never change
 	}
 	h, err := r.head(rel)
 	if err != nil {

cmd/rollcall/redfish.go

@@ -49,8 +49,20 @@ type redfishProfile struct {
 	// Clear: bodies that clear a boot override, tried in order (default: Enabled Disabled)
 	Clear []obj
 	// Auth: "session" logs in through the SessionService (X-Auth-Token); default basic.
-	Auth    string
-	Sources []string
+	Auth string
+	// Firmware (UpdateService, redfish_fw.go). FWPlan builds the vendor's
+	// documented request for a component; without one the standard methods are
+	// tried in FWMethods order (default multipart, simple, push). FWUnsupported
+	// says why Rollcall cannot update this vendor's firmware. FWFormats are the
+	// package files it takes, checked at import.
+	FWPlan        func(fwPlanCtx) (*fwRequest, error)
+	FWMethods     []string
+	FWUnsupported string
+	FWClaims      bool // the plan takes HttpPushUriTargetsBusy; fw-finish gives it back
+	FWFormats     []string
+	FWNotes       string
+	FWSources     []string
+	Sources       []string
 }
 
 // redfishProfiles, most specific first: the AMI-based ones (Gigabyte, ASRock
@@ -168,6 +180,7 @@ type rfClient struct {
 	hc               *http.Client
 	etags            map[string]string // resource -> ETag from its last GET
 	token, session   string            // X-Auth-Token and the session to delete, when logged in
+	pushURIs         map[string]bool   // upload URIs the UpdateService advertised (Lenovo: /mfwupdate)
 }
 
 // login opens a session (profile or extra auth=session); logout closes it.
@@ -228,6 +241,9 @@ func (r *rfClient) resolve(p string) (string, error) {
 		}
 		p = u.RequestURI()
 	}
+	if r.pushURIs[p] && strings.HasPrefix(p, "/") && !strings.Contains(p, "..") {
+		return p, nil // the BMC's own upload URI, on this BMC (outside /redfish/ on some)
+	}
 	if !strings.HasPrefix(p, "/redfish/") || strings.Contains(p, "/../") || strings.HasSuffix(p, "/..") {
 		return "", fmt.Errorf("the BMC pointed at %q, outside /redfish/", p)
 	}

cmd/rollcall/redfish_fw.go

@@ -0,0 +1,666 @@
+package main
+
+// Firmware over Redfish (UpdateService), run where the BMC is reachable (the
+// rack controller). Three operations, each short enough for one rack command:
+//
+//	fw-inventory  what is installed (FirmwareInventory)
+//	fw-start      hand a package to the BMC; returns the task to watch
+//	fw-poll       one look at that task
+//
+// The package reaches the BMC one of three ways, chosen from what the
+// UpdateService offers and the vendor profile's order:
+//
+//	multipart  POST MultipartHttpPushUri: UpdateParameters (JSON) + UpdateFile
+//	simple     UpdateService.SimpleUpdate with an ImageURI the BMC downloads
+//	           from the rack controller (or firmware_base)
+//	push       POST HttpPushUri with the file as the body
+//
+// The rack controller fetches the package from its own cache (filled from the
+// region) and sends it with a known length: many BMC web servers refuse
+// chunked uploads.
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"os"
+	"path"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// fwComponent is one entry of a machine's firmware inventory.
+type fwComponent struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Version    string `json:"version"`
+	Component  string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other
+	Updateable bool   `json:"updateable"`
+	SoftwareID string `json:"software_id,omitempty"`
+}
+
+// fwStartArg is what fw-start receives.
+type fwStartArg struct {
+	URL       string   `json:"url"`               // where the package is (the BMC pulls it, or the rack downloads it to push)
+	Component string   `json:"component"`         // bios | bmc | ... (vendor plans pick targets by it)
+	File      string   `json:"file"`              // its file name
+	Method    string   `json:"method,omitempty"`  // multipart | simple | push; "" = profile order
+	Apply     string   `json:"apply,omitempty"`   // Immediate | OnReset; "" = the BMC's default
+	Targets   []string `json:"targets,omitempty"` // inventory resources to update; usually empty
+}
+
+type fwStartResult struct {
+	Method string `json:"method"`
+	Task   string `json:"task,omitempty"` // task monitor or task resource; "" when the BMC answered synchronously
+	Note   string `json:"note,omitempty"`
+}
+
+// fwTimeouts bound each firmware operation; an upload to a slow BMC takes long.
+var fwTimeouts = map[string]time.Duration{"fw-inventory": 5 * time.Minute, "fw-start": 45 * time.Minute, "fw-poll": 2 * time.Minute, "fw-finish": time.Minute}
+
+type fwPollResult struct {
+	State string `json:"state"` // running | done | failed | unknown
+	Pct   int    `json:"pct"`
+	Msg   string `json:"msg,omitempty"`
+}
+
+// fwKinds sorts inventory entries by id and name, most specific first: a RAID
+// "controller" is not the management controller.
+var fwKinds = []struct {
+	kind string
+	re   *regexp.Regexp
+}{
+	{"bios", regexp.MustCompile(`(?i)\b(bios|uefi|system rom)\b`)},
+	{"raid", regexp.MustCompile(`(?i)\b(raid|perc|smart ?array|megaraid|hba|storage controller|boss)\b`)},
+	{"nic", regexp.MustCompile(`(?i)\b(nic|ethernet|network|connectx|mellanox|infiniband|e810|x710|bcm\d*|broadcom adv)`)},
+	{"psu", regexp.MustCompile(`(?i)\b(psu|power supply)\b`)},
+	{"cpld", regexp.MustCompile(`(?i)\b(cpld|fpga)\b`)},
+	{"drive", regexp.MustCompile(`(?i)\b(drive|disk|ssd|nvme|hdd)\b`)},
+	{"bmc", regexp.MustCompile(`(?i)\b(bmc|idrac|ilo ?\d?|xcc2?|imm2?|ibmc|irmc|cimc|remote access controller|management controller|baseboard management)\b`)},
+}
+
+// fwKind sorts an inventory entry into a component class by its id and name.
+func fwKind(id, name string) string {
+	s := id + " " + name
+	for _, k := range fwKinds {
+		if k.re.MatchString(s) {
+			return k.kind
+		}
+	}
+	return "other"
+}
+
+func firmwareRun(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
+	d, _ := driverByName(c.Driver)
+	if d.Family != "redfish" {
+		return "", fmt.Errorf("firmware updates need a Redfish driver (this BMC uses %s)", orDefault(c.Driver, "ipmi"))
+	}
+	prof := redfishProfileFor(c.Driver)
+	r := newRF(c)
+	r.hc.Timeout = 0 // uploads take minutes; the context bounds each call
+	if err := r.open(ctx, c, prof); err != nil {
+		return "", err
+	}
+	defer r.logout()
+	var out any
+	var err error
+	switch op {
+	case "fw-inventory":
+		out, err = r.fwInventory(ctx)
+	case "fw-start":
+		var a fwStartArg
+		if err := json.Unmarshal([]byte(arg), &a); err != nil {
+			return "", fmt.Errorf("fw-start: %w", err)
+		}
+		out, err = r.fwStart(ctx, c, prof, a)
+	case "fw-poll":
+		out, err = r.fwPoll(ctx, arg)
+	case "fw-finish": // after the task ended: give back what fw-start claimed
+		if prof.FWClaims {
+			r.fwRelease(ctx)
+		}
+		out = obj{"ok": true}
+	default:
+		return "", fmt.Errorf("unknown firmware operation %q", op)
+	}
+	if err != nil {
+		return "", err
+	}
+	b, _ := json.Marshal(out)
+	return string(b), nil
+}
+
+func (r *rfClient) fwInventory(ctx context.Context) ([]fwComponent, error) {
+	var coll struct {
+		Members []struct {
+			ID string `json:"@odata.id"`
+		} `json:"Members"`
+	}
+	if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err != nil {
+		return nil, err
+	}
+	out := []fwComponent{}
+	for i, mem := range coll.Members {
+		if i >= 300 {
+			break
+		}
+		id := path.Base(strings.TrimRight(mem.ID, "/"))
+		// Dell lists what is installed and what was before or is staged
+		if strings.HasPrefix(id, "Previous-") || strings.HasPrefix(id, "Available-") {
+			continue
+		}
+		var e obj
+		if _, err := r.do(ctx, "GET", mem.ID, nil, &e); err != nil {
+			continue
+		}
+		up, _ := e["Updateable"].(bool)
+		fc := fwComponent{ID: str(e["Id"]), Name: str(e["Name"]), Version: str(e["Version"]), Updateable: up, SoftwareID: str(e["SoftwareId"])}
+		if fc.ID == "" {
+			fc.ID = id
+		}
+		fc.Component = fwKind(fc.ID, fc.Name)
+		out = append(out, fc)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Component != out[j].Component {
+			return out[i].Component < out[j].Component
+		}
+		return out[i].ID < out[j].ID
+	})
+	return out, nil
+}
+
+// fwCaps is what an UpdateService offers.
+type fwCaps struct {
+	simple, multipart, push string
+	protocols               []string
+}
+
+func (r *rfClient) fwCaps(ctx context.Context) (fwCaps, error) {
+	var us obj
+	if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil {
+		return fwCaps{}, err
+	}
+	act := asObj(dig(us, "Actions", "#UpdateService.SimpleUpdate"))
+	if r.pushURIs == nil {
+		r.pushURIs = map[string]bool{}
+	}
+	for _, k := range []string{"MultipartHttpPushUri", "HttpPushUri"} {
+		if u := str(us[k]); u != "" && !strings.Contains(u, "://") {
+			r.pushURIs[u] = true
+		}
+	}
+	return fwCaps{
+		simple:    str(act["target"]),
+		protocols: strList(act["TransferProtocol@Redfish.AllowableValues"]),
+		multipart: str(us["MultipartHttpPushUri"]),
+		push:      str(us["HttpPushUri"]),
+	}, nil
+}
+
+// fwRequest is one way to hand a package over, as a vendor documents it.
+type fwRequest struct {
+	Method   string            // multipart | simple | push: how this request carries the package
+	URI      string            // where it goes
+	AltURI   string            // tried when URI answers 404 (paths that moved between firmware generations)
+	Params   obj               // multipart UpdateParameters, or the JSON body of a simple request
+	NoParams bool              // multipart without an UpdateParameters part
+	Parts    []fwPart          // more parts before the file (AMI OemParameters)
+	Field    string            // the file's form field (default UpdateFile)
+	Form     map[string]string // text fields after the file
+	FailText string            // a 2xx answer that contains this is a refusal (KAYTUS "Verification Failed")
+	Claim    bool              // take HttpPushUriTargetsBusy first (Lenovo); fw-finish gives it back
+	Then     *fwRequest        // a second call naming the uploaded file (xFusion, KAYTUS)
+	Tasks    []string          // fixed tasks to watch when the answers name none (KAYTUS)
+}
+
+type fwPart struct {
+	Name string
+	Body obj
+}
+
+// fwPlanCtx is what a vendor plan decides from.
+type fwPlanCtx struct {
+	Component, File, URL, Apply, Password string
+	Vendor                                string   // ServiceRoot Vendor
+	Caps                                  fwCaps   // what the UpdateService offers
+	Inventory                             []string // FirmwareInventory member ids
+}
+
+func (x fwPlanCtx) has(id string) bool { return hasTag(x.Inventory, id) }
+
+func (x fwPlanCtx) first(ids ...string) string {
+	for _, id := range ids {
+		if x.has(id) {
+			return id
+		}
+	}
+	return ""
+}
+
+func (x fwPlanCtx) applyOr(def string) string { return orDefault(x.Apply, def) }
+
+func (r *rfClient) fwStart(ctx context.Context, c *bmcCreds, prof redfishProfile, a fwStartArg) (*fwStartResult, error) {
+	caps, err := r.fwCaps(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("no UpdateService: %w", err)
+	}
+	forced := orDefault(a.Method, c.Extra["fw_method"])
+	if prof.FWUnsupported != "" && forced == "" {
+		return nil, errors.New(prof.FWUnsupported + " (to try a standard method anyway: -method multipart|simple|push)")
+	}
+	if prof.FWPlan != nil && forced == "" {
+		x := fwPlanCtx{Component: a.Component, File: a.File, URL: a.URL, Apply: a.Apply, Password: r.pass, Caps: caps}
+		var root obj
+		if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err == nil {
+			x.Vendor = str(root["Vendor"])
+		}
+		var coll struct {
+			Members []struct {
+				ID string `json:"@odata.id"`
+			} `json:"Members"`
+		}
+		if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err == nil {
+			for _, m := range coll.Members {
+				x.Inventory = append(x.Inventory, path.Base(strings.TrimRight(m.ID, "/")))
+			}
+		}
+		req, err := prof.FWPlan(x)
+		if err != nil {
+			return nil, err
+		}
+		if raw := c.Extra["fw_params"]; raw != "" { // the operator's own UpdateParameters / body
+			var o obj
+			if err := json.Unmarshal([]byte(raw), &o); err != nil {
+				return nil, fmt.Errorf("fw_params is not JSON: %w", err)
+			}
+			req.Params = o
+		}
+		return r.fwExec(ctx, req, a)
+	}
+	order := prof.FWMethods
+	if len(order) == 0 {
+		order = []string{"multipart", "simple", "push"}
+	}
+	if forced != "" {
+		order = []string{forced}
+	}
+	var tried []string
+	for _, m := range order {
+		req := fwStandard(caps, m, a)
+		if req == nil {
+			continue
+		}
+		res, err := r.fwExec(ctx, req, a)
+		if err == nil {
+			return res, nil
+		}
+		// a BMC that does not like this way gets the next one; a refused
+		// package (wrong model, signature, downgrade) stops here
+		if e := asRFError(err); e == nil || (e.code != http.StatusMethodNotAllowed && e.code != http.StatusNotImplemented && e.code != http.StatusUnsupportedMediaType) {
+			return nil, fmt.Errorf("%s: %w", m, err)
+		}
+		tried = append(tried, m+": "+err.Error())
+	}
+	if len(tried) > 0 {
+		return nil, fmt.Errorf("the BMC took no upload method: %s", strings.Join(tried, "; "))
+	}
+	return nil, fmt.Errorf("the BMC's UpdateService offers none of %v", order)
+}
+
+// fwStandard is the DMTF request for a method, nil when the service lacks it.
+func fwStandard(caps fwCaps, method string, a fwStartArg) *fwRequest {
+	switch method {
+	case "multipart":
+		if caps.multipart == "" {
+			return nil
+		}
+		params := obj{"Targets": orEmpty(a.Targets)}
+		if a.Apply != "" {
+			params["@Redfish.OperationApplyTime"] = a.Apply
+		}
+		return &fwRequest{Method: "multipart", URI: caps.multipart, Params: params}
+	case "simple":
+		if caps.simple == "" {
+			return nil
+		}
+		body := obj{"ImageURI": a.URL}
+		if len(caps.protocols) > 0 { // only when the service lists them: others want the scheme in the URI
+			body["TransferProtocol"] = strings.ToUpper(strings.SplitN(a.URL, "://", 2)[0])
+		}
+		if len(a.Targets) > 0 {
+			body["Targets"] = a.Targets
+		}
+		return &fwRequest{Method: "simple", URI: caps.simple, Params: body}
+	case "push":
+		if caps.push == "" {
+			return nil
+		}
+		return &fwRequest{Method: "push", URI: caps.push}
+	}
+	return nil
+}
+
+// fwExec carries out a request (and its follow-up) and names the task to watch.
+func (r *rfClient) fwExec(ctx context.Context, req *fwRequest, a fwStartArg) (*fwStartResult, error) {
+	if req.Claim {
+		var us obj
+		if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil {
+			return nil, err
+		}
+		if us["HttpPushUriTargetsBusy"] == true {
+			return nil, errors.New("the BMC's update service is busy with another upload (HttpPushUriTargetsBusy)")
+		}
+		if _, err := r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": true}); err != nil {
+			return nil, fmt.Errorf("claim the update service: %w", err)
+		}
+	}
+	resp, err := r.fwSend(ctx, req, a)
+	if err != nil {
+		if req.Claim {
+			r.fwRelease(ctx)
+		}
+		return nil, err
+	}
+	res := &fwStartResult{Method: req.Method, Task: fwTaskOf(resp)}
+	if req.Then != nil {
+		tresp, err := r.fwSend(ctx, req.Then, a)
+		if err != nil {
+			return nil, fmt.Errorf("the BMC has the file, but did not start the update: %w", err)
+		}
+		res.Task = fwTaskOf(tresp)
+	}
+	if res.Task == "" && len(req.Tasks) > 0 {
+		res.Task = strings.Join(req.Tasks, ",")
+	}
+	return res, nil
+}
+
+func (r *rfClient) fwRelease(ctx context.Context) {
+	_, _ = r.call(ctx, "GET", "/redfish/v1/UpdateService", nil, "")
+	_, _ = r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": false})
+}
+
+func (r *rfClient) fwSend(ctx context.Context, req *fwRequest, a fwStartArg) (*rfResp, error) {
+	var resp *rfResp
+	var err error
+	send := func(uri string) (*rfResp, error) {
+		switch req.Method {
+		case "simple":
+			if proto := str(req.Params["TransferProtocol"]); proto != "" && len(a.URL) > 0 {
+				// checked here so a service that pulls only over NFS/SCP says so clearly
+				if caps, cerr := r.fwCaps(ctx); cerr == nil && len(caps.protocols) > 0 && !hasTag(caps.protocols, proto) {
+					return nil, &rfError{method: "POST", path: uri, code: http.StatusNotImplemented, msg: fmt.Sprintf("transfer protocols %v, not %s", caps.protocols, proto)}
+				}
+			}
+			return r.call(ctx, "POST", uri, req.Params, "")
+		case "multipart", "push":
+			f, size, ferr := fwFetch(ctx, a.URL)
+			if ferr != nil {
+				return nil, ferr
+			}
+			defer os.Remove(f.Name())
+			defer f.Close()
+			if req.Method == "push" {
+				return r.upload(ctx, uri, "application/octet-stream", f, size, r.etags[uri])
+			}
+			return r.multipartUpload(ctx, uri, req, a.File, f, size)
+		}
+		return nil, fmt.Errorf("unknown method %q", req.Method)
+	}
+	resp, err = send(req.URI)
+	if e := asRFError(err); e != nil && e.code == http.StatusNotFound && req.AltURI != "" {
+		resp, err = send(req.AltURI)
+	}
+	if err == nil && req.FailText != "" && strings.Contains(string(resp.body), req.FailText) {
+		return nil, fmt.Errorf("the BMC refused the package: %s", req.FailText)
+	}
+	return resp, err
+}
+
+func orEmpty(s []string) []string {
+	if s == nil {
+		return []string{}
+	}
+	return s
+}
+
+// fwTaskOf finds the task in an answer: Location, a Task body, or a Dell job id.
+func fwTaskOf(resp *rfResp) string {
+	if resp == nil {
+		return ""
+	}
+	if l := resp.hdr.Get("Location"); l != "" {
+		return l
+	}
+	var t struct {
+		ID   string `json:"@odata.id"`
+		Type string `json:"@odata.type"`
+	}
+	if json.Unmarshal(resp.body, &t) == nil && strings.Contains(t.Type, "Task") {
+		return t.ID
+	}
+	return ""
+}
+
+// fwFetch downloads the package to a temporary file (the rack's cache serves it).
+func fwFetch(ctx context.Context, url string) (*os.File, int64, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, 0, fmt.Errorf("package: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, 0, fmt.Errorf("package %s: %s", url, resp.Status)
+	}
+	f, err := os.CreateTemp("", "rollcall-fw-*")
+	if err != nil {
+		return nil, 0, err
+	}
+	n, err := io.Copy(f, resp.Body)
+	if err == nil {
+		_, err = f.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, 0, err
+	}
+	return f, n, nil
+}
+
+// upload POSTs a body of known length (no chunked encoding).
+func (r *rfClient) upload(ctx context.Context, target, ctype string, body io.Reader, size int64, ifMatch string) (*rfResp, error) {
+	p, err := r.resolve(target)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, "POST", r.base+p, body)
+	if err != nil {
+		return nil, err
+	}
+	req.ContentLength = size
+	req.Header.Set("Content-Type", ctype)
+	req.Header.Set("Accept", "application/json")
+	if ifMatch != "" {
+		req.Header.Set("If-Match", ifMatch)
+	}
+	if r.token != "" {
+		req.Header.Set("X-Auth-Token", r.token)
+	} else if r.user != "" {
+		req.SetBasicAuth(r.user, r.pass)
+	}
+	resp, err := r.hc.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	out := &rfResp{code: resp.StatusCode, hdr: resp.Header, body: data}
+	if resp.StatusCode >= 300 {
+		ids, msg := redfishMessage(data)
+		return out, &rfError{method: "POST", path: p, code: resp.StatusCode, ids: ids, msg: resp.Status + " " + msg}
+	}
+	return out, nil
+}
+
+// multipartUpload sends UpdateParameters, any OEM parts, the file and text
+// fields (DSP0266 "multipart HTTP push", plus what vendors add to it).
+func (r *rfClient) multipartUpload(ctx context.Context, target string, req *fwRequest, name string, f io.Reader, size int64) (*rfResp, error) {
+	var head, after bytes.Buffer
+	mw := multipart.NewWriter(&head)
+	jsonPart := func(n string, v any) {
+		h := textproto.MIMEHeader{}
+		h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q`, n))
+		h.Set("Content-Type", "application/json")
+		w, _ := mw.CreatePart(h)
+		b, _ := json.Marshal(v)
+		_, _ = w.Write(b)
+	}
+	if !req.NoParams {
+		p := req.Params
+		if p == nil {
+			p = obj{"Targets": []string{}}
+		}
+		jsonPart("UpdateParameters", p)
+	}
+	for _, p := range req.Parts {
+		jsonPart(p.Name, p.Body)
+	}
+	fh := textproto.MIMEHeader{}
+	fh.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, orDefault(req.Field, "UpdateFile"), path.Base(name)))
+	fh.Set("Content-Type", "application/octet-stream")
+	_, _ = mw.CreatePart(fh)
+	// text fields and the closing boundary come after the file
+	keys := make([]string, 0, len(req.Form))
+	for k := range req.Form {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		fmt.Fprintf(&after, "\r\n--%s\r\nContent-Disposition: form-data; name=%q\r\n\r\n%s", mw.Boundary(), k, req.Form[k])
+	}
+	fmt.Fprintf(&after, "\r\n--%s--\r\n", mw.Boundary())
+	body := io.MultiReader(&head, f, &after)
+	return r.upload(ctx, target, mw.FormDataContentType(), body, int64(head.Len())+size+int64(after.Len()), r.etags[target])
+}
+
+// fwPoll looks at the task(s) of an update; several (comma-separated) are
+// watched in order and count as done when all are.
+func (r *rfClient) fwPoll(ctx context.Context, tasks string) (*fwPollResult, error) {
+	if tasks == "" {
+		return &fwPollResult{State: "unknown"}, nil
+	}
+	var last *fwPollResult
+	seen := false
+	for _, t := range strings.Split(tasks, ",") {
+		res, err := r.fwPollOne(ctx, t)
+		if err != nil {
+			return nil, err
+		}
+		if res.State == "unknown" { // not (or no longer) listed: the others decide
+			continue
+		}
+		seen = true
+		if res.State != "done" {
+			return res, nil
+		}
+		last = res
+	}
+	if !seen {
+		return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil
+	}
+	return last, nil
+}
+
+var fwScheduledRe = regexp.MustCompile(`(?i)\bscheduled\b|completed successfully`)
+
+func (r *rfClient) fwPollOne(ctx context.Context, task string) (*fwPollResult, error) {
+	resp, err := r.call(ctx, "GET", task, nil, "")
+	if e := asRFError(err); e != nil && e.code == http.StatusNotFound {
+		// iDRAC: while it updates itself the task moves to the job service
+		if i := strings.Index(task, "/TaskService/Tasks/"); i >= 0 {
+			resp, err = r.call(ctx, "GET", "/redfish/v1/JobService/Jobs/"+path.Base(task), nil, "")
+		}
+		if e := asRFError(err); e != nil && e.code == http.StatusNotFound {
+			return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	var t obj
+	_ = json.Unmarshal(resp.body, &t)
+	st := str(t["TaskState"]) + str(t["JobState"])
+	res := &fwPollResult{State: "running", Pct: -1}
+	if p, ok := t["PercentComplete"].(float64); ok {
+		res.Pct = int(p)
+	}
+	for _, v := range asObj(t["Oem"]) { // iBMC TaskPercentage "40%", iRMC TotalProgressPercent
+		o := asObj(v)
+		if p := strings.TrimSuffix(str(o["TaskPercentage"]), "%"); p != "" {
+			res.Pct, _ = strconv.Atoi(p)
+		}
+		if p, ok := o["TotalProgressPercent"].(float64); ok {
+			res.Pct = int(p)
+		}
+	}
+	if ms := toList(t["Messages"]); len(ms) > 0 {
+		res.Msg = str(asObj(ms[len(ms)-1])["Message"])
+	}
+	if res.Msg == "" {
+		res.Msg = str(t["Message"]) // Dell jobs
+	}
+	if res.Pct < 0 { // no percentage on the task: some keep it on the update service
+		res.Pct = 0
+		var us obj
+		if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err == nil {
+			if h := asObj(dig(us, "Oem", "Hpe")); len(h) > 0 { // iLO: State, FlashProgressPercent, Result
+				if p, ok := h["FlashProgressPercent"].(float64); ok {
+					res.Pct = int(p)
+				}
+				if str(h["State"]) == "Error" {
+					res.State, res.Msg = "failed", "iLO update service: "+oneLine(fmt.Sprint(h["Result"]))
+					return res, nil
+				}
+			}
+			if a := asObj(dig(us, "Oem", "AMIUpdateService")); len(a) > 0 { // "3% done."
+				if m := regexp.MustCompile(`(\d+)%`).FindStringSubmatch(str(a["FlashPercentage"])); m != nil {
+					res.Pct, _ = strconv.Atoi(m[1])
+				}
+				if res.Msg == "" {
+					res.Msg = str(a["UpdateStatus"])
+				}
+			}
+		}
+	}
+	switch st {
+	case "Completed":
+		res.State, res.Pct = "done", 100
+	case "Exception", "Killed", "Cancelled", "Failed", "CompletedWithErrors", "Interrupted":
+		res.State = "failed"
+	case "Scheduled", "UserIntervention": // Dell: staged until the host reboots
+		res.State = "done"
+	case "":
+		if resp.code == http.StatusOK && t["TaskState"] == nil && t["JobState"] == nil {
+			res.State = "done" // a task monitor answers with the result once finished
+		}
+	}
+	if res.State == "running" && res.Pct >= 100 && fwScheduledRe.MatchString(res.Msg) {
+		res.State = "done" // staged for the next reboot
+	}
+	return res, nil
+}

cmd/rollcall/redfish_fwplans.go

@@ -0,0 +1,273 @@
+package main
+
+// Firmware update plans per vendor: the request each one documents for a
+// component. Sources are on each entry; "oss" ones come from the vendor's own
+// tools (Ansible collections, PowerShell modules, SDKs) where its API guide is
+// not public. An operator can replace a plan's UpdateParameters with
+// fw_params={...} on the BMC record, or skip the plan with fw_method=.
+
+import (
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+)
+
+type fwVendor struct {
+	plan        func(fwPlanCtx) (*fwRequest, error)
+	methods     []string
+	claims      bool
+	unsupported string
+	formats     []string
+	notes       string
+	sources     []string
+}
+
+var fwVendors = map[string]fwVendor{
+	"idrac": {
+		// MultipartUpload; older iDRAC 9 without it pulls with SimpleUpdate.
+		// OnReset by default: BIOS-type jobs wait for a reboot anyway, and an
+		// iDRAC or CPLD update must not surprise anyone.
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			apply := x.applyOr("OnReset")
+			if x.Caps.multipart != "" {
+				return &fwRequest{Method: "multipart", URI: x.Caps.multipart,
+					Params: obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply, "Oem": obj{}}}, nil
+			}
+			if x.Caps.simple != "" {
+				return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: simpleBody(x, obj{"@Redfish.OperationApplyTime": apply})}, nil
+			}
+			return nil, errors.New("this iDRAC offers neither MultipartHttpPushUri nor SimpleUpdate")
+		},
+		formats: []string{".exe", ".d9", ".d7", ".pm"},
+		notes:   "Dell Update Packages (Windows .EXE). BIOS, RAID, NIC jobs apply at the next reboot (-reboot); iDRAC, Diags apply by themselves; a CPLD update does a virtual A/C cycle.",
+		sources: []string{
+			"https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareMultipartUploadREDFISH.py",
+			"https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareSimpleUpdateTransferProtocolREDFISH.py",
+		},
+	},
+	"ilo": {
+		// SimpleUpdate over HTTP(S) flashes iLO .bin and UEFI .signed.flash;
+		// a Smart Component (.fwpkg) answers 200 and flashes nothing, so
+		// import extracts the flashable file from it.
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			if x.Caps.simple == "" {
+				return nil, errors.New("this iLO offers no SimpleUpdate")
+			}
+			return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": x.URL}}, nil
+		},
+		formats: []string{".bin", ".signed.flash", ".flash", ".fwpkg"},
+		notes:   "iLO firmware flashes at once (iLO restarts); UEFI/BIOS is staged and flashed at the next reboot (-reboot). Components only a runtime agent can flash (some NICs) need SUM/iSUT in the OS. Downgrades follow the iLO downgrade policy.",
+		sources: []string{
+			"https://servermanagementportal.ext.hpe.com/docs/references_and_material/blogposts/firmware_updates/part3/firmware_update_part3",
+			"https://servermanagementportal.ext.hpe.com/docs/redfishservices/ilos/supplementdocuments/updateservice/",
+		},
+	},
+	"xcc": {
+		// /mfwupdate, after taking the push service (HttpPushUriTargetsBusy).
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			params := obj{"Targets": []string{}}
+			if x.Apply != "" {
+				params["@Redfish.OperationApplyTime"] = x.Apply
+			}
+			if x.Caps.multipart != "" {
+				return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params, Claim: true}, nil
+			}
+			if x.Caps.push != "" {
+				return &fwRequest{Method: "push", URI: x.Caps.push, Claim: true}, nil
+			}
+			return nil, errors.New("this XCC offers no push update URI")
+		},
+		claims:  true,
+		formats: []string{".uxz", ".zip", ".bin"},
+		notes:   "Update XCC before UEFI (Lenovo). One push at a time; the busy flag is released after the update.",
+		sources: []string{
+			"https://pubs.lenovo.com/xcc-restapi/simple_update_firmware_post_multi_httppush",
+			"https://pubs.lenovo.com/xcc2-restapi/prop_firmware_update_service_get",
+			"https://pubs.lenovo.com/xcc/updating_firmware_overview",
+		},
+	},
+	"supermicro": {
+		// /UpdateService/upload with the documented targets and preserve options.
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			uri := orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload")
+			apply := x.applyOr("Immediate")
+			var params obj
+			switch x.Component {
+			case "bios":
+				params = obj{"Targets": []string{"/redfish/v1/Systems/1/Bios"}, "@Redfish.OperationApplyTime": apply,
+					"Oem": obj{"Supermicro": obj{"BIOS": obj{"PreserveME": true, "PreserveNVRAM": true, "PreserveSMBIOS": true, "BackupBIOS": false}}}}
+			case "bmc":
+				params = obj{"Targets": []string{"/redfish/v1/Managers/1"}, "@Redfish.OperationApplyTime": apply,
+					"Oem": obj{"Supermicro": obj{"BMC": obj{"PreserveCfg": true, "PreserveSdr": true, "PreserveSsl": true, "BackupBMC": true}}}}
+			case "cpld":
+				id := ""
+				for _, i := range x.Inventory {
+					if strings.HasPrefix(i, "Motherboard_CPLD_") && id == "" {
+						id = i
+					}
+				}
+				if id == "" {
+					return nil, errors.New("the BMC lists no Motherboard_CPLD_* firmware")
+				}
+				params = obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + id}, "@Redfish.OperationApplyTime": apply}
+			default: // NIC, storage AOC, PSU: the BMC picks the device (SFT-DCMS-SINGLE licence)
+				params = obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply}
+			}
+			return &fwRequest{Method: "multipart", URI: uri, Params: params}, nil
+		},
+		notes: "A BIOS update powers the host off first. Gen 12/13 boards take other BIOS preserve options (PreserveOA, PreserveSETUPCONF, ...): set fw_params on the BMC if the BMC refuses these. Redfish updates need the SFT-OOB-LIC or SFT-DCMS-SINGLE licence; NIC/storage/PSU need DCMS. A CPLD update resets the BMC.",
+		sources: []string{
+			"https://www.supermicro.com/manuals/other/RedfishUserGuide.pdf",
+			"https://github.com/bmc-toolbox/bmclib/blob/main/providers/supermicro/x12.go",
+		},
+	},
+	"gigabyte":   {plan: amiPlan, formats: []string{".hpm", ".rcu", ".ima_enc", ".ima", ".rbu", ".bin"}, notes: amiNotes, sources: []string{"https://download.gigabyte.com/FileList/Manual/server_manual_redfish_v1.11.0.pdf", "GIGABYTE Firmware Upgrade Guide v0.04 (2022)"}},
+	"ami":        {plan: amiPlan, notes: amiNotes, sources: []string{"https://download2.mitaccomputing.com/pub/doc/MiTAC_AST2600_Redfish_API_Doc_v.1_051024.pdf", "https://support.blackcoretech.com/support/solutions/articles/70000680445-update-bios-bmc-via-redfish-multipart-push"}},
+	"asrockrack": {plan: amiPlan, notes: amiNotes + " ASRock Rack's own Redfish firmware notes could not be read: this is the AMI MegaRAC request, check it on one machine first.", sources: []string{"https://download.asrock.com/TSD/SMU/Manual/ASRock%20Rack%20Server%20Management_v2.8.0.pdf"}},
+	"openbmc": {
+		// update-multipart; the BMC image applies at once (it reboots), the rest OnReset.
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			params := obj{"Targets": []string{}}
+			if x.Component == "bmc" {
+				params = obj{"Targets": []string{"/redfish/v1/Managers/bmc"}, "@Redfish.OperationApplyTime": x.applyOr("Immediate")}
+			} else if x.Apply != "" {
+				params["@Redfish.OperationApplyTime"] = x.Apply
+			}
+			if x.Caps.multipart != "" {
+				return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params}, nil
+			}
+			if x.Caps.push != "" && x.Component == "bmc" {
+				return &fwRequest{Method: "push", URI: x.Caps.push}, nil
+			}
+			return nil, errors.New("this OpenBMC offers no multipart update (SimpleUpdate is off in bmcweb by default)")
+		},
+		formats: []string{".tar"},
+		notes:   "Signed image tarballs whose MANIFEST MachineName matches the BMC.",
+		sources: []string{"https://github.com/openbmc/bmcweb/blob/master/redfish-core/lib/update_service.hpp", "https://github.com/openbmc/docs/blob/master/REDFISH-cheatsheet.md"},
+	},
+	"xfusion": {
+		// upload to the BMC's /tmp/web, then SimpleUpdate naming the file
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			if x.Component != "bmc" && x.Component != "bios" && x.Component != "cpld" {
+				return nil, fmt.Errorf("iBMC updates BMC, BIOS and CPLD out of band; %s firmware goes through Smart Provisioning in-band (not supported)", x.Component)
+			}
+			if x.Caps.simple == "" {
+				return nil, errors.New("this iBMC offers no SimpleUpdate")
+			}
+			return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "file",
+				Then: &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": "/tmp/web/" + path.Base(x.File)}}}, nil
+		},
+		formats: []string{".hpm"},
+		notes:   "The BMC restarts after a BMC update.",
+		sources: []string{
+			"https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_manage_file.py",
+			"https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_outband_fw_update.py",
+		},
+	},
+	"kaytus": {
+		// upload (with the BMC password), then an OEM SimpleUpdate; the tasks have fixed names
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			oem := obj{"PreserveConf": true}
+			var tasks []string
+			switch x.Component {
+			case "bmc":
+				oem["FlashItem"], tasks = "OpenBMC_ENC", []string{"/redfish/v1/TaskService/Tasks/BMC"}
+			case "bios":
+				oem["FlashItem"], oem["BiosFlash"] = "BIOS", "Both"
+				tasks = []string{"/redfish/v1/TaskService/Tasks/BIOS0", "/redfish/v1/TaskService/Tasks/BIOS1"}
+			default:
+				return nil, fmt.Errorf("KAYTUS: BMC and BIOS only (not %s)", x.Component)
+			}
+			return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "fwimage",
+				Form: map[string]string{"CurrentPassword": x.Password, "EncryptFlag": "False"}, FailText: "Verification Failed",
+				Then:  &fwRequest{Method: "simple", URI: orDefault(x.Caps.simple, "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"), Params: obj{"Oem": obj{"Public": oem}}},
+				Tasks: tasks}, nil
+		},
+		formats: []string{".hpm", ".bin"},
+		notes:   "No public Redfish guide: this is what KAYTUS's own SDK (ksManage) does on X3-generation BMCs; older generations use a non-Redfish API. A BIOS update with the host on waits for a power cycle (-reboot); the BMC restarts after a BMC update.",
+		sources: []string{"https://github.com/KSManageOSS/ksManage/blob/HEAD/ksmanage_sdk/interface/CommonX3.py", "https://github.com/ieisystem/kaytus.ksmanage"},
+	},
+	"irmc": {
+		// OEM actions per component, file in the "data" part
+		plan: func(x fwPlanCtx) (*fwRequest, error) {
+			p := "Fsas" // FTS on S5/S6 (Vendor ts_fujitsu or none), Fsas on newer firmware
+			if x.Vendor == "" || strings.EqualFold(x.Vendor, "ts_fujitsu") {
+				p = "FTS"
+			}
+			switch x.Component {
+			case "bmc":
+				return &fwRequest{Method: "multipart", URI: "/redfish/v1/Managers/iRMC/Actions/" + p + "Manager.FWUpdate",
+					AltURI: "/redfish/v1/Managers/iRMC/Actions/Oem/" + p + "Manager.FWUpdate", NoParams: true, Field: "data"}, nil
+			case "bios":
+				return &fwRequest{Method: "multipart", URI: "/redfish/v1/Systems/0/Bios/Actions/Oem/" + p + "Bios.BiosUpdate", NoParams: true, Field: "data"}, nil
+			}
+			return nil, fmt.Errorf("iRMC: iRMC and BIOS only (not %s)", x.Component)
+		},
+		formats: []string{".bin", ".upc"},
+		notes:   "iRMC .bin, BIOS .UPC. With the host on, a BIOS update waits for a reboot (-reboot); the iRMC restarts after its own update.",
+		sources: []string{"https://github.com/fujitsu/ansible-irmc-integration/blob/HEAD/plugins/modules/irmc_fwbios_update.py", "https://github.com/mmurayama/fujitsu-redfish-samples/blob/master/scripts/update_irmc_firmware.py"},
+	},
+	"cimc": {
+		unsupported: "Cisco documents CIMC firmware updates over Redfish only as SimpleUpdate pulls over SCP (and HSU/HUU from NFS, CIFS or HTTPS shares); Rollcall serves packages over HTTP",
+		sources:     []string{"https://www.cisco.com/c/en/us/td/docs/unified_computing/ucs/c/sw/api/4_3/b-cisco-ucs-c-series-servers-rest-api-programmer-s-guide-release-4-3.pdf"},
+	},
+}
+
+const amiNotes = "MegaRAC: the BMC restarts after a BMC flash and accepts no logins while flashing; do not use its web UI meanwhile."
+
+// amiPlan: /UpdateService/upload with UpdateParameters (the inventory target)
+// and OemParameters (ImageType); .hpm packages have their own targets.
+func amiPlan(x fwPlanCtx) (*fwRequest, error) {
+	hpm := strings.HasSuffix(strings.ToLower(x.File), ".hpm")
+	var target, typ string
+	switch x.Component {
+	case "bmc":
+		if hpm {
+			target, typ = "BMC", "HPM_BMC"
+		} else {
+			target, typ = orDefault(x.first("BMCImage1", "BMC"), "BMC"), "BMC"
+		}
+	case "bios":
+		if hpm {
+			target, typ = "BIOS2", "HPM_BIOS"
+		} else {
+			target, typ = "BIOS", "BIOS"
+		}
+	case "cpld":
+		target, typ = "MB_CPLD1", "MB_CPLD"
+	default:
+		return nil, fmt.Errorf("MegaRAC: BMC, BIOS and CPLD (not %s)", x.Component)
+	}
+	if len(x.Inventory) > 0 && !x.has(target) {
+		return nil, fmt.Errorf("the BMC lists no firmware %s (it has %s)", target, strings.Join(x.Inventory, ", "))
+	}
+	params := obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + target}}
+	if x.Apply != "" {
+		params["@Redfish.OperationApplyTime"] = x.Apply
+	}
+	return &fwRequest{Method: "multipart", URI: orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload"), Params: params,
+		Parts: []fwPart{{Name: "OemParameters", Body: obj{"ImageType": typ}}}}, nil
+}
+
+// simpleBody is a SimpleUpdate body for the package URL, plus extra fields.
+func simpleBody(x fwPlanCtx, extra obj) obj {
+	b := obj{"ImageURI": x.URL}
+	if len(x.Caps.protocols) > 0 {
+		b["TransferProtocol"] = strings.ToUpper(strings.SplitN(x.URL, "://", 2)[0])
+	}
+	for k, v := range extra {
+		b[k] = v
+	}
+	return b
+}
+
+func init() {
+	for i := range redfishProfiles {
+		p := &redfishProfiles[i]
+		if v, ok := fwVendors[p.Driver]; ok {
+			p.FWPlan, p.FWMethods, p.FWClaims, p.FWUnsupported = v.plan, v.methods, v.claims, v.unsupported
+			p.FWFormats, p.FWNotes, p.FWSources = v.formats, v.notes, v.sources
+		}
+	}
+}

cmd/rollcall/server.go

@@ -201,6 +201,12 @@ func (a *App) routes() http.Handler {
 	mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
 	// disk-based deployment
 	mux.HandleFunc("GET /api/v1/os", a.adminAuth(a.handleOS))
+	mux.HandleFunc("GET /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
+	mux.HandleFunc("POST /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
+	mux.HandleFunc("DELETE /api/v1/firmware/{pkg}", a.adminAuth(a.handleFirmwareRemove))
+	mux.HandleFunc("GET /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
+	mux.HandleFunc("POST /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
+	mux.HandleFunc("POST /api/v1/machines/{id}/firmware/update", a.adminAuth(a.handleFirmwareUpdate))
 	mux.HandleFunc("POST /api/v1/machines/{id}/deploy", a.adminAuth(a.handleDeploy))
 	mux.HandleFunc("POST /api/v1/machines/{id}/release", a.adminAuth(a.handleRelease))
 	mux.HandleFunc("GET /api/v1/machines/{id}/jobs", a.adminAuth(a.handleJobs))
@@ -282,6 +288,7 @@ func (a *App) routes() http.Handler {
 	mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
 	mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
 	mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir))))
+	mux.HandleFunc("GET /firmware/{pkg}/{file}", a.handleFirmwareFile) // BMCs download packages (no listing)
 	// embedded web UI
 	sub, _ := fs.Sub(webFS, "web")
 	mux.Handle("GET /", a.uiOverHTTPS(uiHeaders(http.FileServer(http.FS(sub)))))
@@ -795,6 +802,7 @@ func serve(cfgPath string) error {
 		return fmt.Errorf("dns: %w", err)
 	}
 	go app.jobWatchdog()
+	go app.resumeFirmware()
 	handler := app.routes()
 	if cfg.TLSListen != "" {
 		names := append([]string{}, cfg.TLSNames...)

cmd/rollcall/store.go

@@ -239,6 +239,9 @@ func openStore(ctx context.Context, url string) (*Store, error) {
 	if _, err := pool.Exec(ctx, dnsRecordsSchema); err != nil {
 		return nil, fmt.Errorf("schema: %w", err)
 	}
+	if _, err := pool.Exec(ctx, firmwareSchema); err != nil {
+		return nil, fmt.Errorf("schema: %w", err)
+	}
 	return &Store{pool: pool}, nil
 }
 

cmd/rollcall/web/app.js

@@ -125,7 +125,7 @@ function render() {
     case 'network': main.innerHTML = pageNetwork(); wireNetwork(); break;
     case 'tags': main.innerHTML = pageTags(); wireTags(); break;
     case 'profiles': main.innerHTML = pageProfiles(); wireProfiles(); break;
-    case 'images': main.innerHTML = pageImages(); break;
+    case 'images': main.innerHTML = pageImages(); wireImages(); break;
     default: main.innerHTML = pageOverview(); wireOverview();
   }
 }
@@ -453,7 +453,7 @@ function jobsTable(jobs, withMachine) {
 function pageMachine(d, tab) {
   const s = d.summary, inv = d.inventory || {}, sys = inv.system || {}, cpu = inv.cpu || {}, mem = inv.memory || {}, st = inv.storage || {}, b = d.bmc || {};
   const m = S.machines.find(x => x.id === s.id) || s;
-  const tabs = ['overview', 'hardware', 'network', 'bmc', 'jobs', 'events', 'raw'];
+  const tabs = ['overview', 'hardware', 'network', 'bmc', 'firmware', 'jobs', 'events', 'raw'];
   let body = '';
   if (tab === 'overview') {
     const job = s.job || (d.jobs || [])[0]; const p = (job && job.progress) || {}; const active = job && (job.state === 'pending' || job.state === 'running');
@@ -501,7 +501,8 @@ function pageMachine(d, tab) {
       ['Source', esc(b.source)], ['Address', `<span class="mono">${esc(b.address)}${b.port && b.port !== 623 ? ':' + esc(b.port) : ''}</span>`], ['MAC', `<span class="mono">${esc(b.mac)}</span>`], ['User', esc(b.user)], ['Password', d.has_bmc_secret ? '<span class="ok">stored encrypted on the server</span>' : '<span class="bad">none</span>'], ['Configured', b.configured ? '<span class="ok">yes</span>' : '<span class="bad">no</span>'], ['Verified out-of-band', b.verified_oob === true ? '<span class="ok">yes</span>' : b.verified_oob === false ? '<span class="bad">no</span>' : ''], ['Channel', esc(b.channel)], ['LAN', b.lan ? `<span class="mono">${esc(JSON.stringify(b.lan))}</span>` : ''], ['Firmware', esc((b.mc_info || {})['Firmware Revision'])], ['Manufacturer', esc((b.mc_info || {})['Manufacturer Name'])], ['Errors', (b.errors || []).length ? `<span class="bad">${esc(b.errors.join('; '))}</span>` : '']]))}
       ${(b.users_before || []).length ? card('User table before commissioning', table(['ID', 'Name', 'Priv', 'IPMI'], b.users_before.map(u => [esc(u.id), esc(u.name), esc(u.priv), u.ipmi_msg ? 'yes' : 'no']))) : ''}
       ${b.fru ? card('FRU', `<pre class="json">${esc(JSON.stringify(b.fru, null, 1))}</pre>`) : ''}</div>`;
-  } else if (tab === 'jobs') body = jobsTable(d.jobs || [], false);
+  } else if (tab === 'firmware') body = `<div class="actions" style="margin-bottom:12px"><button class="btn" id="fw-read" title="read the firmware inventory from the BMC (Redfish)">Read from BMC</button><span class="muted">Packages come from the repository (OS images page); the update runs on rack controller ${esc(s.rack_controller || '')}</span></div><div id="fwbox" class="muted">loading…</div>`;
+  else if (tab === 'jobs') body = jobsTable(d.jobs || [], false);
   else if (tab === 'events') body = table(['Time', 'Event', 'Detail'], (d.events || []).slice().reverse().map(e => [esc(when(e.ts)), esc(e.event), `<span class="mono">${esc(e.detail)}</span>`]));
   else body = `<pre class="json">${esc(JSON.stringify(d, null, 1))}</pre>`;
 
@@ -522,6 +523,7 @@ function pageMachine(d, tab) {
 function wireMachine(d) {
   const id = d.summary.id, ids = [id];
   if (route().tab === 'network') wireNetPlan(id);
+  if (route().tab === 'firmware') loadFirmware(id, false);
   wireMenus();
   $$('#dpower [data-item]').forEach(b => b.onclick = () => act.power(ids, b.dataset.item));
   $$('#dboot [data-item]').forEach(b => b.onclick = () => act.bootdev(ids, b.dataset.item));
@@ -535,6 +537,54 @@ function wireMachine(d) {
   $$('[data-untag]').forEach(a => a.onclick = ev => { ev.preventDefault(); each(ids, 'untag', async () => (await api('POST', `/api/v1/machines/${id}/tags`, { remove: [a.dataset.untag] })).effective.join(' ')); });
 }
 
+// ---------------------------------------------------------------- firmware
+async function loadFirmware(id, read) {
+  const box = $('#fwbox'); if (!box) return;
+  const btn = $('#fw-read'); if (btn) btn.onclick = () => loadFirmware(id, true);
+  if (read) box.innerHTML = '<span class="muted">reading the inventory from the BMC…</span>';
+  let d;
+  try { d = await api(read ? 'POST' : 'GET', `/api/v1/machines/${id}/firmware`); } catch (e) { box.innerHTML = `<span class="bad">${esc(e.message)}</span>`; return; }
+  const inv = (d.inventory || {}).components || [];
+  const offers = d.offers || [];
+  box.className = '';
+  box.innerHTML = `<div class="grid">
+    ${card(`Installed · ${inv.length} components${d.inventory && d.inventory.at && inv.length ? ` · read ${esc(ago(d.inventory.at))} ago` : ''}`, inv.length ? table(['Component', 'Name', 'Version', 'Updateable'], inv.map(c => [esc(c.component), esc(c.name), `<span class="mono">${esc(c.version)}</span>`, c.updateable ? 'yes' : '<span class="muted">no</span>'])) : '<span class="muted">not read yet: Read from BMC</span>', true)}
+    ${card(`Repository for ${esc(d.driver || '?')} ${esc(d.model || '')}`, offers.length ? table(['Component', 'Version', 'Installed now', '', ''], offers.map(o => [esc(o.package.component), `<span class="mono">${esc(o.package.version)}</span>`, `<span class="mono">${esc(o.installed || '—')}</span>`, o.same ? '<span class="ok">installed</span>' : '<span class="warn">differs</span>', `<button class="btn sm" data-fwpkg="${esc(o.package.id)}">${o.same ? 'Reinstall…' : 'Update…'}</button>`])) : '<span class="muted">no package in the repository for this vendor and model</span>', true)}
+  </div>`;
+  $$('[data-fwpkg]').forEach(b => b.onclick = () => fwUpdate(id, offers.find(o => o.package.id === b.dataset.fwpkg).package));
+}
+async function fwUpdate(id, p) {
+  const v = await dialog(`Update ${p.component} to ${p.version}`, [
+    { name: 'apply', label: 'Apply', type: 'select', value: '', options: [['', "the BMC's default"], ['OnReset', 'at the next reset'], ['Immediate', 'immediately']] },
+    { name: 'reboot', label: 'Power-cycle the host afterwards and verify the version', type: 'checkbox', value: p.component === 'bios' },
+    { name: 'force', label: 'Send even if vendor or model does not match', type: 'checkbox', value: false },
+  ], `${p.file} (${human(p.size)}). The BMC checks the package itself; an update cannot be cancelled once the BMC has it. A BMC update restarts the BMC for a few minutes.`, 'Start update');
+  if (!v) return;
+  try { const r = await api('POST', `/api/v1/machines/${id}/firmware/update`, { package: p.id, apply: v.apply, reboot: v.reboot, force: v.force }); toast(`firmware job ${r.job} started`); await loadDetail(id); } catch (e) { toast(e.message, true); }
+}
+function wireImages() {
+  const el = $('#fwrepo'); if (!el) return;
+  const show = () => api('GET', '/api/v1/firmware').then(ps => {
+    el.className = '';
+    el.innerHTML = ps.length ? table(['Package', 'Vendor', 'Component', 'Version', 'Models', 'Size', 'Imported', ''], ps.map(p => [`<span class="mono">${esc(p.id)}</span><br><span class="muted">${esc(p.file)}</span>`, esc(p.vendor), esc(p.component), `<span class="mono">${esc(p.version)}</span>`, esc((p.models || []).join(', ') || 'any'), human(p.size), esc(when(p.imported)), `<button class="btn sm" data-fwrm="${esc(p.id)}">remove</button>`])) : '<span class="muted">no packages yet</span>';
+    $$('[data-fwrm]').forEach(b => b.onclick = async () => { if (confirm(`Remove ${b.dataset.fwrm}?`)) { try { await api('DELETE', '/api/v1/firmware/' + b.dataset.fwrm); show(); } catch (e) { toast(e.message, true); } } });
+  }).catch(e => { el.innerHTML = `<span class="bad">${esc(e.message)}</span>`; });
+  show();
+  $('#fwimport').onclick = async () => {
+    const drivers = await api('GET', '/api/v1/power-drivers').then(ds => ds.filter(d => d.family === 'redfish' && d.name !== 'redfish').map(d => [d.name, d.description]), () => []);
+    const v = await dialog('Import a firmware package', [
+      { name: 'url', label: 'Download URL', placeholder: 'https://dl.dell.com/FOLDER.../BIOS_XXXX_WN64_2.19.1.EXE' },
+      { name: 'vendor', label: 'Vendor (BMC driver)', type: 'select', options: drivers },
+      { name: 'component', label: 'Component', type: 'select', options: ['bios', 'bmc', 'nic', 'raid', 'cpld', 'psu', 'drive', 'other'].map(c => [c, c]) },
+      { name: 'version', label: 'Version', placeholder: 'as the component reports it once installed' },
+      { name: 'models', label: 'Models', placeholder: 'R650, R750 (empty: any model of this vendor)' },
+      { name: 'sha256', label: 'SHA-256', placeholder: "from the vendor's download page (recommended)" },
+    ], 'The region downloads the package; rack controllers serve it to the BMCs. A local file: rollcall firmware import FILE on the region.', 'Import');
+    if (!v) return;
+    try { const p = await api('POST', '/api/v1/firmware', { url: v.url.trim(), vendor: v.vendor, component: v.component, version: v.version.trim(), models: v.models.split(',').map(x => x.trim()).filter(Boolean), sha256: v.sha256.trim() }); toast(`imported ${p.id}`); show(); } catch (e) { toast(e.message, true); }
+  };
+}
+
 // ---------------------------------------------------------------- jobs
 function pageJobs() {
   const active = S.jobs.filter(j => j.state === 'pending' || j.state === 'running'), rest = S.jobs.filter(j => !(j.state === 'pending' || j.state === 'running'));
@@ -638,6 +688,9 @@ function wireProfiles() {
   api('GET', '/api/v1/images').then(imgs => { const el = $('#rootimgs'); if (el) el.innerHTML = table(['Image', 'Complete', 'Squashfs', 'Built'], imgs.map(i => [esc(i.name), i.complete ? '<span class="ok">yes</span>' : '<span class="bad">no</span>', human(i['filesystem.squashfs']), esc(when(i.built))])); }).catch(() => { });
 }
 function pageImages() {
+  return pageOSImages() + `<div class="card wide" style="margin-top:16px"><h3>Firmware packages <button class="btn sm" id="fwimport" style="margin-left:8px">Import…</button></h3><div id="fwrepo" class="muted">loading…</div><p class="muted" style="margin:8px 0 0">Updates run from a machine's Firmware tab, or <code>rollcall firmware update MACHINE|tag:T PACKAGE</code>.</p></div>`;
+}
+function pageOSImages() {
   return `<div class="page-head"><h1>OS images</h1><span class="sub">disk images for deployment · refreshed nightly by <code>rollcall os sync</code> from the vendors' checksum files</span></div>
   ${table(['OS', 'Current version', 'Family', 'Compressed', 'Disk needed', 'Imported', 'Track', 'Keep', 'Versions', 'Status'], S.os.map(i => { const c = i.current || {}; const s = i.source || {}; return [`<b>${esc(i.name)}</b>${s.description ? `<br><span class="muted">${esc(s.description)}</span>` : ''}`, `<span class="mono">${esc(c.version || '—')}</span>`, esc(s.family || c.family), c.size ? human(c.size) : '', c.raw_size ? human(c.raw_size) : '', c.imported ? `${esc(when(c.imported))} <span class="muted">(${ago(c.imported)})</span>` : '', esc(s.track || ''), s.keep || '', (i.versions || []).map(v => `<span class="mono${v.current ? '' : ' muted'}">${esc(v.version)}</span>`).join('<br>'), i.warning ? `<span class="warn">${esc(i.warning)}</span>` : '<span class="ok">ok</span>']; }))}
   <p class="muted" style="margin-top:10px">Pinned sources keep <code>current</code> until <code>rollcall os promote NAME VERSION</code>; a deployment always uses the exact version it started with. Import a local file with <code>rollcall os import NAME FILE</code>.</p>`;

lab/pve-vbmc/pve_redfish.py

@@ -11,9 +11,19 @@ idrac / ilo / generic drivers see what real hardware shows:
 
 Config: the pve-vbmc config.json; a bmcs entry may carry "redfish": "idrac"|"ilo"|"generic".
 Basic auth with the same user/password as the IPMI side. Self-signed TLS.
+
+Firmware (UpdateService), for testing rollcall's firmware jobs: idrac takes
+multipart uploads and SimpleUpdate, ilo only SimpleUpdate (it downloads the
+ImageURI itself), generic multipart. A "package" is any file with a line
+"version=X"; a name with bios/rom/flash in it is the BIOS, anything else the
+BMC. The BMC version changes when its task completes; the BIOS one at the next
+reset of the VM, as on real servers. Versions are kept in
+/var/lib/pve-vbmc/fw-<vmid>.json.
 """
 import argparse
 import base64
+import email
+import email.policy
 import ipaddress
 import json
 import logging
@@ -35,11 +45,16 @@ log = logging.getLogger("pve-redfish")
 PROFILES = {
     "idrac": {"vendor": "Dell", "manufacturer": "Dell Inc.", "model": "PowerEdge R760 (pve)", "id": "System.Embedded.1",
               "resets": ["On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"],
-              "actioninfo": False, "etag": True},
+              "actioninfo": False, "etag": True, "fw": {"multipart": "/redfish/v1/UpdateService/MultipartUpload", "simple": True,
+                                                        "ids": {"bmc": "Installed-25227", "bios": "Installed-159"},
+                                                        "names": {"bmc": "Integrated Dell Remote Access Controller", "bios": "BIOS"}}},
     "ilo": {"vendor": "HPE", "manufacturer": "HPE", "model": "ProLiant DL380 Gen11 (pve)", "id": "1",
-            "resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False},
+            "resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False,
+            "fw": {"multipart": "", "simple": True, "ids": {"bmc": "1", "bios": "2"}, "names": {"bmc": "iLO 6", "bios": "System ROM"}}},
     "generic": {"vendor": "", "manufacturer": "QEMU", "model": "Standard PC (pve)", "id": "1",
-                "resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False},
+                "resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False,
+                "fw": {"multipart": "/redfish/v1/UpdateService/update-multipart", "simple": False, "ids": {"bmc": "bmc", "bios": "bios"},
+                       "names": {"bmc": "BMC firmware", "bios": "BIOS"}}},
 }
 
 
@@ -82,7 +97,67 @@ class Pve:
         return order
 
 
-def make_handler(pve, prof, user, password):
+class Firmware:
+    """Installed versions, staged BIOS and update tasks of one emulated BMC."""
+
+    def __init__(self, vmid, prof):
+        self.path = f"/var/lib/pve-vbmc/fw-{vmid}.json"
+        self.prof = prof["fw"]
+        self.lock = threading.Lock()
+        self.tasks = {}
+        try:
+            self.v = json.load(open(self.path))
+        except (OSError, ValueError):
+            self.v = {"bmc": "1.0.0", "bios": "1.0.0", "staged_bios": ""}
+
+    def save(self):
+        os.makedirs(os.path.dirname(self.path), exist_ok=True)
+        json.dump(self.v, open(self.path, "w"))
+
+    def inventory(self):
+        return [{"@odata.id": f"/redfish/v1/UpdateService/FirmwareInventory/{self.prof['ids'][c]}-{self.v[c]}", "Id": f"{self.prof['ids'][c]}-{self.v[c]}",
+                 "Name": self.prof["names"][c], "Version": self.v[c], "Updateable": True} for c in ("bmc", "bios")]
+
+    def start(self, name, data):
+        """Begin flashing; returns the task id."""
+        comp = "bios" if any(w in name.lower() for w in ("bios", "rom", "flash")) else "bmc"
+        ver = ""
+        for line in data.decode(errors="replace").splitlines():
+            if line.startswith("version="):
+                ver = line.split("=", 1)[1].strip()
+        with self.lock:
+            tid = str(len(self.tasks) + 1)
+            self.tasks[tid] = {"state": "Running", "pct": 0, "msg": f"flashing {comp} {ver}"}
+        threading.Thread(target=self.flash, args=(tid, comp, ver), daemon=True).start()
+        log.info("firmware task %s: %s -> %s (%s)", tid, comp, ver or "?", name)
+        return tid
+
+    def flash(self, tid, comp, ver):
+        for pct in (10, 30, 60, 90):
+            time.sleep(4)
+            self.tasks[tid]["pct"] = pct
+        t = self.tasks[tid]
+        if not ver:
+            t.update(state="Exception", msg="the image is not valid for this system")
+            return
+        with self.lock:
+            if comp == "bmc":
+                self.v["bmc"] = ver
+                t.update(state="Completed", pct=100, msg=f"BMC firmware {ver} installed")
+            else:
+                self.v["staged_bios"] = ver
+                t.update(state="Completed", pct=100, msg=f"BIOS {ver} staged; it applies at the next reboot")
+            self.save()
+
+    def reset(self):
+        with self.lock:
+            if self.v.get("staged_bios"):
+                log.info("reset: BIOS %s applied", self.v["staged_bios"])
+                self.v["bios"], self.v["staged_bios"] = self.v["staged_bios"], ""
+                self.save()
+
+
+def make_handler(pve, prof, user, password, fw):
     sysid = prof["id"]
     syspath = f"/redfish/v1/Systems/{sysid}"
     state = {"override": "None", "enabled": "Disabled", "etag": 1}
@@ -150,6 +225,24 @@ def make_handler(pve, prof, user, password):
                 return self.send(200, self.system(), hdr)
             if p == syspath + "/ResetActionInfo":
                 return self.send(200, {"Parameters": [{"Name": "ResetType", "Required": True, "AllowableValues": prof["resets"]}]})
+            if p == "/redfish/v1/UpdateService":
+                us = {"@odata.id": p, "ServiceEnabled": True, "FirmwareInventory": {"@odata.id": p + "/FirmwareInventory"}}
+                if fw.prof["multipart"]:
+                    us["MultipartHttpPushUri"] = fw.prof["multipart"]
+                if fw.prof["simple"]:
+                    us["Actions"] = {"#UpdateService.SimpleUpdate": {"target": p + "/Actions/UpdateService.SimpleUpdate",
+                                                                    "TransferProtocol@Redfish.AllowableValues": ["HTTP", "HTTPS"]}}
+                return self.send(200, us)
+            if p == "/redfish/v1/UpdateService/FirmwareInventory":
+                return self.send(200, {"Members": [{"@odata.id": e["@odata.id"]} for e in fw.inventory()]})
+            if p.startswith("/redfish/v1/UpdateService/FirmwareInventory/"):
+                for e in fw.inventory():
+                    if e["@odata.id"] == p:
+                        return self.send(200, e)
+            if p.startswith("/redfish/v1/TaskService/Tasks/"):
+                t = fw.tasks.get(p.rsplit("/", 1)[1])
+                if t:
+                    return self.send(200, {"@odata.id": p, "TaskState": t["state"], "PercentComplete": t["pct"], "Messages": [{"Message": t["msg"]}]})
             self.error(404, "not found")
 
         def do_PATCH(self):
@@ -170,10 +263,33 @@ def make_handler(pve, prof, user, password):
             state["etag"] += 1
             self.send(204)
 
+        def task(self, tid):
+            loc = f"/redfish/v1/TaskService/Tasks/{tid}"
+            self.send(202, {"@odata.id": loc, "@odata.type": "#Task.v1_4_0.Task", "TaskState": "Running"}, {"Location": loc})
+
         def do_POST(self):
             if not self.authed():
                 return
-            if self.path.rstrip("/") != syspath + "/Actions/ComputerSystem.Reset":
+            p = self.path.split("?")[0].rstrip("/")
+            if fw.prof["multipart"] and p == fw.prof["multipart"]:
+                n = int(self.headers.get("Content-Length") or 0)
+                if n <= 0:
+                    return self.error(411, "Content-Length required")
+                raw = self.rfile.read(n)
+                msg = email.message_from_bytes(b"Content-Type: " + self.headers["Content-Type"].encode() + b"\r\n\r\n" + raw, policy=email.policy.default)
+                for part in msg.iter_parts():
+                    if part.get_param("name", header="content-disposition") == "UpdateFile":
+                        return self.task(fw.start(part.get_filename() or "image", part.get_payload(decode=True)))
+                return self.error(400, "no UpdateFile part")
+            if fw.prof["simple"] and p == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate":
+                uri = self.body().get("ImageURI", "")
+                try:
+                    r = requests.get(uri, timeout=60)
+                    r.raise_for_status()
+                except requests.RequestException as e:
+                    return self.error(400, f"cannot download {uri}: {e}")
+                return self.task(fw.start(uri.rsplit("/", 1)[-1], r.content))
+            if p != syspath + "/Actions/ComputerSystem.Reset":
                 return self.error(404, "not found")
             t = self.body().get("ResetType", "")
             if t not in prof["resets"]:
@@ -190,6 +306,8 @@ def make_handler(pve, prof, user, password):
                 pve.post("status/shutdown", forceStop=1, timeout=60)
             elif t in ("ForceRestart", "GracefulRestart", "PowerCycle"):
                 pve.post("status/reset" if on else "status/start")
+            if t in ("On", "ForceRestart", "GracefulRestart", "PowerCycle", "PushPowerButton"):
+                fw.reset()  # a staged BIOS applies when the host starts again
             self.send(204)
 
     return H
@@ -223,7 +341,7 @@ def main():
     addr = entry["address"]
     user = entry.get("user", cfg.get("user", "admin"))
     password = entry.get("password", cfg.get("password", "password"))
-    handler = make_handler(Pve(cfg, args.vmid), prof, user, password)
+    handler = make_handler(Pve(cfg, args.vmid), prof, user, password, Firmware(args.vmid, prof))
     ip = ipaddress.ip_address(addr)
     cls = Server6 if ip.version == 6 else ThreadingHTTPServer
     srv = cls((addr, int(entry.get("redfish_port", 443))), handler)