From b01d8e2b4dc62cdfa3e30418b44fb85ad9123ed1 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 29 Oct 2026 19:22:01 +0100 Subject: [PATCH] Firmware updates over Redfish, per vendor A firmware repository on the region (firmware_dir, served under /firmware//, cached by rack controllers for BMCs to pull), each machine's firmware inventory (FirmwareInventory, classified into bios/bmc/nic/raid/cpld/psu/drive), and update jobs (kind "firmware") that the region drives with short rack commands: fw-start hands the package over, fw-poll watches the task (tolerating a BMC that restarts), fw-finish releases what was claimed, fw-inventory verifies the version. Running updates resume after a region restart; they cannot be cancelled once the BMC has the package. Standard methods (multipart, SimpleUpdate pull, HttpPushUri) with known Content-Length, plus vendor plans from their documentation or their own tools: Dell MultipartUpload (OnReset), HPE SimpleUpdate with .fwpkg extracted at import, Lenovo /mfwupdate with HttpPushUriTargetsBusy, Supermicro /upload with targets and preserve options, GIGABYTE/AMI/ ASRock Rack /upload with OemParameters ImageType, OpenBMC update-multipart, xFusion upload to /tmp/web then SimpleUpdate, KAYTUS upload then OEM SimpleUpdate with fixed tasks, Fujitsu iRMC OEM actions; Cisco CIMC is refused clearly (SCP only). fw_params and fw_method on the BMC record override a plan. CLI rollcall firmware list|import|rm|show|update, API, a Firmware tab and a repository card in the UI. The lab Redfish emulator gains an UpdateService (BMC applies at once, BIOS at the next reset). Co-Authored-By: Claude Opus 5.5 --- cmd/rollcall/config.go | 9 +- cmd/rollcall/deploy.go | 6 +- cmd/rollcall/firmware.go | 868 ++++++++++++++++++++++++++++++++ cmd/rollcall/firmware_test.go | 289 +++++++++++ cmd/rollcall/fwplans_test.go | 306 +++++++++++ cmd/rollcall/ipmi.go | 19 + cmd/rollcall/main.go | 7 + cmd/rollcall/power.go | 5 + cmd/rollcall/rackctl.go | 7 +- cmd/rollcall/rackd.go | 6 +- cmd/rollcall/redfish.go | 20 +- cmd/rollcall/redfish_fw.go | 666 ++++++++++++++++++++++++ cmd/rollcall/redfish_fwplans.go | 273 ++++++++++ cmd/rollcall/server.go | 8 + cmd/rollcall/store.go | 3 + cmd/rollcall/web/app.js | 59 ++- lab/pve-vbmc/pve_redfish.py | 130 ++++- 17 files changed, 2663 insertions(+), 18 deletions(-) create mode 100644 cmd/rollcall/firmware.go create mode 100644 cmd/rollcall/firmware_test.go create mode 100644 cmd/rollcall/fwplans_test.go create mode 100644 cmd/rollcall/redfish_fw.go create mode 100644 cmd/rollcall/redfish_fwplans.go diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index 4dc1fff..d60e34f 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -31,8 +31,10 @@ type Config struct { ClusterVars map[string]any `json:"cluster_vars"` // available to overlay templates as .Vars (profile vars override) // disk-based deployment - OSDir string `json:"os_dir"` // ///{disk.raw.zst,meta.json}, served under /os/ - OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync` + OSDir string `json:"os_dir"` // ///{disk.raw.zst,meta.json}, served under /os/ + // FirmwareDir: //{meta.json,}, served under /firmware// + FirmwareDir string `json:"firmware_dir"` + OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync` // region/rack: this server runs an embedded rack controller named RackName; // remote rack controllers (rollcall rack) authenticate with RackSecret @@ -103,6 +105,9 @@ func loadConfig(path string) (*Config, error) { if c.OSDir == "" { c.OSDir = filepath.Join(filepath.Dir(c.ImageDir), "os") } + if c.FirmwareDir == "" { + c.FirmwareDir = filepath.Join(filepath.Dir(c.ImageDir), "firmware") + } if c.OSSources == "" { c.OSSources = "/etc/rollcall/os-sources.json" } diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index 5f30789..ffb3793 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -626,6 +626,10 @@ func (a *App) handleJobCancel(w http.ResponseWriter, r *http.Request) { fail(w, 404, fmt.Errorf("no active job")) return } + if j.Kind == "firmware" { + fail(w, 409, errors.New("a firmware update cannot be stopped from here once the BMC has it (it would leave the component half-flashed); it ends by itself")) + return + } _ = a.store.FinishJob(r.Context(), j.ID, "cancelled", "cancelled by operator") a.afterJob(r.Context(), m, j, "cancelled", "cancelled by operator") writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "state": "cancelled"}) @@ -792,7 +796,7 @@ func (a *App) jobStartTimeout() time.Duration { func staleJobs(active map[string]*Job, cutoff time.Time) []*Job { var out []*Job for _, j := range active { - if j.State == "pending" && j.Created.Before(cutoff) { + if j.State == "pending" && j.Kind != "firmware" && j.Created.Before(cutoff) { // firmware jobs have their own watch out = append(out, j) } } diff --git a/cmd/rollcall/firmware.go b/cmd/rollcall/firmware.go new file mode 100644 index 0000000..81c9db0 --- /dev/null +++ b/cmd/rollcall/firmware.go @@ -0,0 +1,868 @@ +package main + +// Firmware: a repository of vendor packages on the region, each machine's +// firmware inventory read from its BMC, and update jobs. +// +// //meta.json + the package file, served under /firmware/ +// +// A package is for one vendor profile (the Redfish driver: idrac, ilo, ...), +// one component (bios, bmc, nic, raid, cpld, psu, drive, other), a version, +// and optionally models. An update is a job (kind "firmware") that the region +// drives with short rack commands: fw-start hands the package to the BMC, +// fw-poll watches the BMC's task until it ends, fw-inventory reads the +// versions afterwards. The task is kept in the job, so a region restart +// resumes watching instead of losing the update. + +import ( + "archive/zip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +type FWPackage struct { + ID string `json:"id"` + Vendor string `json:"vendor"` // a Redfish driver: idrac, ilo, xcc, supermicro, ... + Component string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other + Version string `json:"version"` + Models []string `json:"models,omitempty"` // model substrings it is for; none = any of the vendor's + File string `json:"file"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Source string `json:"source,omitempty"` + Notes string `json:"notes,omitempty"` + Imported time.Time `json:"imported"` +} + +var fwComponents = []string{"bios", "bmc", "nic", "raid", "cpld", "psu", "drive", "other"} + +var fwIDRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,80}$`) +var fwFileRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,120}$`) + +const firmwareSchema = ` +CREATE TABLE IF NOT EXISTS firmware_inventory ( + machine_id text PRIMARY KEY REFERENCES machines(id) ON DELETE CASCADE, + at timestamptz NOT NULL DEFAULT now(), + components jsonb NOT NULL +); +` + +// ---------------------------------------------------------------- repository + +func fwList(dir string) ([]FWPackage, error) { + ents, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return []FWPackage{}, nil + } + if err != nil { + return nil, err + } + out := []FWPackage{} + for _, e := range ents { + b, err := os.ReadFile(filepath.Join(dir, e.Name(), "meta.json")) + if err != nil { + continue + } + var p FWPackage + if json.Unmarshal(b, &p) == nil && p.ID == e.Name() { + out = append(out, p) + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Vendor != b.Vendor { + return a.Vendor < b.Vendor + } + if a.Component != b.Component { + return a.Component < b.Component + } + return a.Imported.After(b.Imported) + }) + return out, nil +} + +func fwGet(dir, id string) (FWPackage, error) { + if !fwIDRe.MatchString(id) { + return FWPackage{}, fmt.Errorf("no firmware package %q", id) + } + b, err := os.ReadFile(filepath.Join(dir, id, "meta.json")) + if err != nil { + return FWPackage{}, fmt.Errorf("no firmware package %q", id) + } + var p FWPackage + return p, json.Unmarshal(b, &p) +} + +// fwCheck validates a package description against what Rollcall knows. +func fwCheck(p *FWPackage) error { + p.Vendor, p.Component = strings.ToLower(p.Vendor), strings.ToLower(p.Component) + if d, ok := driverByName(p.Vendor); !ok || d.Family != "redfish" { + var names []string + for _, r := range redfishProfiles { + names = append(names, r.Driver) + } + return fmt.Errorf("vendor is a Redfish driver: %s", strings.Join(names, ", ")) + } + if !hasTag(fwComponents, p.Component) { + return fmt.Errorf("component is one of %s", strings.Join(fwComponents, ", ")) + } + if strings.TrimSpace(p.Version) == "" || strings.ContainsAny(p.Version, "\r\n") { + return errors.New("version is required") + } + if !fwFileRe.MatchString(p.File) { + return fmt.Errorf("file name %q: letters, digits and ._+- only", p.File) + } + if fs := redfishProfileFor(p.Vendor).FWFormats; len(fs) > 0 { + ok := false + for _, ext := range fs { + ok = ok || strings.HasSuffix(strings.ToLower(p.File), ext) + } + if !ok { + return fmt.Errorf("%s takes %s packages, not %s", p.Vendor, strings.Join(fs, " "), p.File) + } + } + return nil +} + +// fwImport copies (or downloads) a package into the repository. +func fwImport(ctx context.Context, dir, src string, p FWPackage, wantSum string) (*FWPackage, error) { + if p.File == "" { + p.File = path.Base(strings.SplitN(src, "?", 2)[0]) + } + if err := fwCheck(&p); err != nil { + return nil, err + } + var in io.ReadCloser + if strings.HasPrefix(src, "https://") || strings.HasPrefix(src, "http://") { + req, err := http.NewRequestWithContext(ctx, "GET", src, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + resp.Body.Close() + return nil, fmt.Errorf("%s: %s", src, resp.Status) + } + in = resp.Body + } else { + f, err := os.Open(src) + if err != nil { + return nil, err + } + in = f + } + defer in.Close() + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + tmp, err := os.MkdirTemp(dir, ".import-") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmp) + f, err := os.Create(filepath.Join(tmp, p.File)) + if err != nil { + return nil, err + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(f, h), in) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return nil, err + } + p.Size, p.SHA256 = n, hex.EncodeToString(h.Sum(nil)) + if wantSum != "" && !strings.EqualFold(wantSum, p.SHA256) { + return nil, fmt.Errorf("sha256 %s, expected %s", p.SHA256, wantSum) + } + if p.Vendor == "ilo" && strings.HasSuffix(strings.ToLower(p.File), ".fwpkg") { + if err := fwpkgExtract(tmp, &p); err != nil { + return nil, err + } + } + if p.Source == "" { + p.Source = src + } + p.Imported = time.Now().UTC() + slug := strings.Trim(regexp.MustCompile(`[^a-z0-9.]+`).ReplaceAllString(strings.ToLower(p.Vendor+"-"+p.Component+"-"+p.Version), "-"), "-") + if len(slug) > 60 { + slug = slug[:60] + } + p.ID = slug + "-" + p.SHA256[:8] + mb, _ := json.MarshalIndent(p, "", " ") + if err := os.WriteFile(filepath.Join(tmp, "meta.json"), append(mb, '\n'), 0o644); err != nil { + return nil, err + } + if err := os.Chmod(tmp, 0o755); err != nil { + return nil, err + } + dst := filepath.Join(dir, p.ID) + if _, err := os.Stat(dst); err == nil { + return &p, nil // the same package again + } + if err := os.Rename(tmp, dst); err != nil { + return nil, err + } + return &p, nil +} + +// fwpkgExtract: iLO's SimpleUpdate flashes the iLO .bin or UEFI .signed.flash +// inside an HPE Smart Component; given the .fwpkg it answers 200 and flashes +// nothing. The .fwpkg is a zip: the one flashable file is kept instead. +func fwpkgExtract(dir string, p *FWPackage) error { + src := filepath.Join(dir, p.File) + zr, err := zip.OpenReader(src) + if err != nil { + return fmt.Errorf("%s is not a readable .fwpkg: %w", p.File, err) + } + defer zr.Close() + var cands []*zip.File + for _, f := range zr.File { + n := strings.ToLower(path.Base(f.Name)) + if !f.FileInfo().IsDir() && (strings.HasSuffix(n, ".bin") || strings.HasSuffix(n, ".flash")) && fwFileRe.MatchString(path.Base(f.Name)) { + cands = append(cands, f) + } + } + if len(cands) != 1 { + var names []string + for _, f := range cands { + names = append(names, f.Name) + } + return fmt.Errorf("%s holds %d files iLO flashes directly (%s); a component only UEFI or an OS agent installs needs SUM/iLOrest", p.File, len(cands), strings.Join(names, ", ")) + } + in, err := cands[0].Open() + if err != nil { + return err + } + defer in.Close() + name := path.Base(cands[0].Name) + out, err := os.Create(filepath.Join(dir, name)) + if err != nil { + return err + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(in, 1<<30)) + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + _ = os.Remove(src) + p.Notes = strings.TrimSpace(p.Notes + fmt.Sprintf(" extracted from %s (sha256 %s)", p.File, p.SHA256)) + p.File, p.Size, p.SHA256 = name, n, hex.EncodeToString(h.Sum(nil)) + return nil +} + +func fwRemove(dir, id string) error { + if !fwIDRe.MatchString(id) { + return fmt.Errorf("no firmware package %q", id) + } + if _, err := os.Stat(filepath.Join(dir, id, "meta.json")); err != nil { + return fmt.Errorf("no firmware package %q", id) + } + return os.RemoveAll(filepath.Join(dir, id)) +} + +// fwFor: the packages for this machine (vendor, model), newest first per component. +func fwFor(pkgs []FWPackage, vendor, model string) []FWPackage { + var out []FWPackage + for _, p := range pkgs { + if p.Vendor != vendor { + continue + } + if len(p.Models) > 0 { + ok := false + for _, m := range p.Models { + ok = ok || strings.Contains(strings.ToLower(model), strings.ToLower(m)) + } + if !ok { + continue + } + } + out = append(out, p) + } + return out +} + +// ---------------------------------------------------------------- inventory + +type fwInventory struct { + At time.Time `json:"at"` + Components []fwComponent `json:"components"` +} + +func (s *Store) FirmwareInventory(ctx context.Context, id string) (*fwInventory, error) { + var inv fwInventory + var b []byte + err := s.pool.QueryRow(ctx, `SELECT at, components FROM firmware_inventory WHERE machine_id=$1`, id).Scan(&inv.At, &b) + if err != nil { + return nil, err + } + return &inv, json.Unmarshal(b, &inv.Components) +} + +func (s *Store) PutFirmwareInventory(ctx context.Context, id string, cs []fwComponent) error { + b, _ := json.Marshal(cs) + _, err := s.pool.Exec(ctx, `INSERT INTO firmware_inventory(machine_id, at, components) VALUES ($1, now(), $2) + ON CONFLICT (machine_id) DO UPDATE SET at=now(), components=$2`, id, b) + return err +} + +func (a *App) readFirmware(ctx context.Context, m *Machine) ([]fwComponent, error) { + out, err := a.firmwareVia(ctx, m, "fw-inventory", "") + if err != nil { + return nil, err + } + var cs []fwComponent + if err := json.Unmarshal([]byte(out), &cs); err != nil { + return nil, fmt.Errorf("inventory: %w", err) + } + return cs, a.store.PutFirmwareInventory(ctx, m.ID, cs) +} + +// bmcVendor is the machine's Redfish driver and model, as probed. +func bmcVendor(m *Machine) (driver, model string) { + var b bmcRecord + _ = json.Unmarshal(m.BMC, &b) + var rf struct { + Model string `json:"model"` + } + var raw obj + _ = json.Unmarshal(m.BMC, &raw) + if x, ok := raw["redfish"]; ok { + bb, _ := json.Marshal(x) + _ = json.Unmarshal(bb, &rf) + } + model = rf.Model + if model == "" { + model = deref(m.Product) + } + return b.Driver, model +} + +// fwCompare marks, per component, whether a package in the repository is newer than what is installed. +type fwOffer struct { + Package FWPackage `json:"package"` + Installed string `json:"installed"` // the version now on the component ("" when not seen) + Same bool `json:"same"` // the package's version is what is installed +} + +func fwOffers(pkgs []FWPackage, inv []fwComponent) []fwOffer { + var out []fwOffer + for _, p := range pkgs { + o := fwOffer{Package: p} + for _, c := range inv { + if c.Component == p.Component { + if o.Installed == "" { + o.Installed = c.Version + } + if fwSameVersion(c.Version, p.Version) { + o.Installed, o.Same = c.Version, true + } + } + } + out = append(out, o) + } + return out +} + +// fwSameVersion compares loosely: "2.19.1" matches "2.19.1", "v2.19.1" and "2.19.1 (Build 4)". +func fwSameVersion(installed, pkg string) bool { + norm := func(s string) string { return strings.TrimPrefix(strings.ToLower(strings.TrimSpace(s)), "v") } + i, p := norm(installed), norm(pkg) + return i == p || (p != "" && (strings.HasPrefix(i, p+" ") || strings.HasPrefix(i, p+"(") || strings.HasPrefix(i, p+"-"))) +} + +// ---------------------------------------------------------------- jobs + +type fwSpec struct { + Package string `json:"package"` + Component string `json:"component"` + Version string `json:"version"` + Apply string `json:"apply,omitempty"` // Immediate | OnReset + Method string `json:"method,omitempty"` // forced upload method + Reboot bool `json:"reboot,omitempty"` // power-cycle the host when the update waits for a reset +} + +type fwProgress struct { + Phase string `json:"phase"` // uploading | flashing | rebooting | verifying + Pct int `json:"pct"` + Msg string `json:"msg,omitempty"` + Method string `json:"method,omitempty"` + Task string `json:"task,omitempty"` +} + +func (a *App) setFWProgress(ctx context.Context, j *Job, p fwProgress) { + b, _ := json.Marshal(p) + _ = a.store.JobProgress(ctx, j.ID, b) +} + +// firmwareURL is where the BMC downloads the package: the machine's rack +// controller, or firmware_base on the BMC record when BMCs reach another address. +func (a *App) firmwareURL(m *Machine, p FWPackage) string { + base := a.urlFor(m) + var b bmcRecord + if json.Unmarshal(m.BMC, &b) == nil && b.Extra["firmware_base"] != "" { + base = strings.TrimRight(b.Extra["firmware_base"], "/") + } + return base + "/firmware/" + p.ID + "/" + p.File +} + +func (a *App) startFirmware(ctx context.Context, m *Machine, p FWPackage, sp fwSpec, force bool) (*Job, error) { + drv, model := bmcVendor(m) + if d, _ := driverByName(drv); d.Family != "redfish" { + return nil, fmt.Errorf("%s: firmware updates need a Redfish BMC driver (now %s): rollcall bmc %s -probe", m.ID, orDefault(drv, "ipmi"), m.ID) + } + if !force { + if p.Vendor != drv { + return nil, fmt.Errorf("%s is a %s package; this BMC is %s (-force to send it anyway)", p.ID, p.Vendor, drv) + } + if len(fwFor([]FWPackage{p}, drv, model)) == 0 { + return nil, fmt.Errorf("%s is for models %v, this is %q (-force to send it anyway)", p.ID, p.Models, model) + } + } + if j, err := a.store.ActiveJob(ctx, m.ID); err == nil { + return nil, fmt.Errorf("job %s (%s, %s) is still active", j.ID, j.Kind, j.State) + } + sp.Package, sp.Component, sp.Version = p.ID, p.Component, p.Version + specB, _ := json.Marshal(sp) + j := &Job{ID: "j-" + randomToken(5), MachineID: m.ID, Kind: "firmware", State: "pending", Spec: specB} + if err := a.store.CreateJob(ctx, j, "", nil); err != nil { + return nil, err + } + a.setFWProgress(ctx, j, fwProgress{Phase: "uploading", Msg: "handing " + p.File + " to the BMC"}) + _ = a.store.AddEvent(ctx, m.ID, "firmware requested", fmt.Sprintf("job=%s %s %s %s apply=%s", j.ID, p.Component, p.Version, p.File, orDefault(sp.Apply, "default"))) + go a.runFirmware(j.ID, m.ID, false) + return j, nil +} + +// runFirmware drives one update to its end; resume picks up a task already started. +func (a *App) runFirmware(jobID, machineID string, resume bool) { + ctx := context.Background() + j, err := a.store.Job(ctx, jobID) + if err != nil { + return + } + m, err := a.store.Get(ctx, machineID) + if err != nil { + return + } + var sp fwSpec + _ = json.Unmarshal(j.Spec, &sp) + var pr fwProgress + _ = json.Unmarshal(j.Progress, &pr) + finish := func(state, msg string) { + _ = a.store.FinishJob(ctx, j.ID, state, map[bool]string{true: "", false: msg}[state == "done"]) + ev := "firmware " + state + _ = a.store.AddEvent(ctx, m.ID, ev, fmt.Sprintf("job=%s %s %s: %s", j.ID, sp.Component, sp.Version, msg)) + log.Printf("%s: firmware %s %s %s: %s", m.ID, sp.Component, sp.Version, state, msg) + } + if !resume || pr.Task == "" && pr.Phase == "uploading" { + if resume { // the region went away while the upload was running: the BMC may or may not have it + finish("failed", "the region restarted during the upload; check the BMC, then try again") + return + } + p, err := fwGet(a.cfg.FirmwareDir, sp.Package) + if err != nil { + finish("failed", err.Error()) + return + } + arg, _ := json.Marshal(fwStartArg{URL: a.firmwareURL(m, p), File: p.File, Component: p.Component, Method: sp.Method, Apply: sp.Apply}) + out, err := a.firmwareVia(ctx, m, "fw-start", string(arg)) + if err != nil { + finish("failed", err.Error()) + return + } + var res fwStartResult + _ = json.Unmarshal([]byte(out), &res) + pr = fwProgress{Phase: "flashing", Method: res.Method, Task: res.Task, Msg: "the BMC has the package"} + a.setFWProgress(ctx, j, pr) + } + // watch the task; a BMC flashing itself drops off the network for minutes + if pr.Task != "" && pr.Phase == "flashing" { + var lastOK time.Time = time.Now() + for { + out, err := a.firmwareVia(ctx, m, "fw-poll", pr.Task) + if err != nil { + if time.Since(lastOK) > 25*time.Minute { + finish("failed", "the BMC has not answered for 25 minutes: "+err.Error()) + return + } + pr.Msg = "BMC not answering (it may be restarting): " + oneLine(err.Error()) + a.setFWProgress(ctx, j, pr) + time.Sleep(20 * time.Second) + continue + } + lastOK = time.Now() + var pl fwPollResult + _ = json.Unmarshal([]byte(out), &pl) + pr.Pct, pr.Msg = pl.Pct, pl.Msg + a.setFWProgress(ctx, j, pr) + if pl.State == "failed" { + _, _ = a.firmwareVia(ctx, m, "fw-finish", "") + finish("failed", orDefault(pl.Msg, "the BMC reported the update failed")) + return + } + if pl.State == "done" || pl.State == "unknown" { + break + } + if time.Since(*orNow(j.Started)) > 3*time.Hour { + finish("failed", "the BMC task did not end within 3 hours") + return + } + time.Sleep(15 * time.Second) + } + } + _, _ = a.firmwareVia(ctx, m, "fw-finish", "") // e.g. Lenovo's busy flag + // a component that applies at the next reset (BIOS, most NICs/RAID) waits for it + if sp.Reboot && pr.Phase != "verifying" { + pr.Phase, pr.Msg = "rebooting", "power cycle so the update applies" + a.setFWProgress(ctx, j, pr) + if _, err := a.power(ctx, m, "cycle", ""); err != nil { + finish("failed", "update staged, but the power cycle failed: "+err.Error()) + return + } + } + pr.Phase, pr.Msg = "verifying", "reading the firmware inventory" + a.setFWProgress(ctx, j, pr) + deadline := time.Now().Add(20 * time.Minute) // servers take minutes to POST, BMCs to come back + for { + inv, err := a.readFirmware(ctx, m) + if err == nil { + for _, c := range inv { + if c.Component == sp.Component && fwSameVersion(c.Version, sp.Version) { + finish("done", "installed "+c.Name+" "+c.Version) + return + } + } + } + if time.Now().After(deadline) || (!sp.Reboot && err == nil && pr.Task != "") { + break + } + time.Sleep(60 * time.Second) + } + if sp.Reboot { + finish("failed", "the inventory does not show "+sp.Version+" after the update and a reboot") + return + } + // staged: most components switch at the next reset of the host + finish("done", "accepted by the BMC; "+sp.Version+" is not in the inventory yet (applies at the next reboot, or rerun with -reboot)") +} + +func orNow(t *time.Time) *time.Time { + if t == nil { + n := time.Now() + return &n + } + return t +} + +// resumeFirmware watches the updates that were running when the region stopped. +func (a *App) resumeFirmware() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + active, err := a.store.ActiveJobs(ctx) + if err != nil { + return + } + for _, j := range active { + if j.Kind == "firmware" { + log.Printf("%s: resuming firmware job %s", j.MachineID, j.ID) + go a.runFirmware(j.ID, j.MachineID, true) + } + } +} + +// ---------------------------------------------------------------- HTTP + +func (a *App) handleFirmwareRepo(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + pkgs, err := fwList(a.cfg.FirmwareDir) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, pkgs) + case http.MethodPost: // import from a URL: {"url", "vendor", "component", "version", "models", "sha256", "notes"} + var req struct { + FWPackage + URL string `json:"url"` + SHA256 string `json:"sha256"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !(strings.HasPrefix(req.URL, "https://") || strings.HasPrefix(req.URL, "http://")) { + fail(w, 400, errors.New("url (http or https), vendor, component and version are required")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute) + defer cancel() + p, err := fwImport(ctx, a.cfg.FirmwareDir, req.URL, req.FWPackage, req.SHA256) + if err != nil { + fail(w, 400, err) + return + } + log.Printf("firmware %s imported by %s from %s", p.ID, a.who(r), req.URL) + writeJSON(w, 200, p) + } +} + +func (a *App) handleFirmwareRemove(w http.ResponseWriter, r *http.Request) { + if err := fwRemove(a.cfg.FirmwareDir, r.PathValue("pkg")); err != nil { + fail(w, 404, err) + return + } + writeJSON(w, 200, obj{"removed": r.PathValue("pkg")}) +} + +// handleFirmwareFile serves one package file (no listings): BMCs download from here. +func (a *App) handleFirmwareFile(w http.ResponseWriter, r *http.Request) { + p, err := fwGet(a.cfg.FirmwareDir, r.PathValue("pkg")) + if err != nil || r.PathValue("file") != p.File { + http.NotFound(w, r) + return + } + http.ServeFile(w, r, filepath.Join(a.cfg.FirmwareDir, p.ID, p.File)) +} + +// handleMachineFirmware: GET the stored inventory and what the repository offers; POST ?refresh reads it again. +func (a *App) handleMachineFirmware(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 404, err) + return + } + if r.Method == http.MethodPost { + ctx, cancel := context.WithTimeout(r.Context(), 6*time.Minute) + defer cancel() + if _, err := a.readFirmware(ctx, m); err != nil { + fail(w, 502, err) + return + } + } + inv, err := a.store.FirmwareInventory(r.Context(), m.ID) + if err != nil { + inv = &fwInventory{Components: []fwComponent{}} + } + drv, model := bmcVendor(m) + pkgs, _ := fwList(a.cfg.FirmwareDir) + offers := fwOffers(fwFor(pkgs, drv, model), inv.Components) + if offers == nil { + offers = []fwOffer{} + } + writeJSON(w, 200, obj{"id": m.ID, "driver": drv, "model": model, "inventory": inv, "offers": offers}) +} + +// handleFirmwareUpdate starts one update: {"package", "apply", "reboot", "method", "force"}. +func (a *App) handleFirmwareUpdate(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 404, err) + return + } + var req struct { + fwSpec + Force bool `json:"force"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Package == "" { + fail(w, 400, errors.New("package is required")) + return + } + switch req.Apply { + case "", "Immediate", "OnReset": + default: + fail(w, 400, errors.New("apply is Immediate or OnReset")) + return + } + switch req.Method { + case "", "multipart", "simple", "push": + default: + fail(w, 400, errors.New("method is multipart, simple or push")) + return + } + p, err := fwGet(a.cfg.FirmwareDir, req.Package) + if err != nil { + fail(w, 404, err) + return + } + j, err := a.startFirmware(r.Context(), m, p, req.fwSpec, req.Force) + if err != nil { + fail(w, 409, err) + return + } + log.Printf("%s: firmware %s requested by %s (job %s)", m.ID, p.ID, a.who(r), j.ID) + writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "package": p.ID, "component": p.Component, "version": p.Version}) +} + +// ---------------------------------------------------------------- CLI + +func firmwareCmd(args []string) error { + if len(args) == 0 { + args = []string{"list"} + } + switch args[0] { + case "import": + if len(args) < 2 { + usage() + } + fs := newFlags("firmware import") + vendor := fs.String("vendor", "", "Redfish driver the package is for: idrac, ilo, xcc, supermicro, xfusion, kaytus, irmc, cimc, gigabyte, asrockrack, openbmc, ami") + comp := fs.String("component", "", strings.Join(fwComponents, " | ")) + ver := fs.String("version", "", "the version the component reports once installed") + models := fs.String("models", "", "comma-separated model substrings it is for (default: any)") + sum := fs.String("sha256", "", "expected SHA-256 (from the vendor's download page)") + notes := fs.String("notes", "", "notes") + _ = fs.Parse(args[2:]) + p := FWPackage{Vendor: *vendor, Component: *comp, Version: *ver, Notes: *notes} + if *models != "" { + p.Models = strings.Split(*models, ",") + } + src := args[1] + if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { // a local file: on the region host + cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")) + if err != nil { + return fmt.Errorf("importing a file needs the region's config (run on the region, or give an https URL): %w", err) + } + got, err := fwImport(context.Background(), cfg.FirmwareDir, src, p, *sum) + if err != nil { + return err + } + fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256) + return nil + } + c, err := newClient() + if err != nil { + return err + } + var got FWPackage + req := obj{"url": src, "vendor": p.Vendor, "component": p.Component, "version": p.Version, "models": p.Models, "sha256": *sum, "notes": p.Notes} + if err := c.do("POST", "/api/v1/firmware", req, &got); err != nil { + return err + } + fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256) + return nil + } + c, err := newClient() + if err != nil { + return err + } + switch args[0] { + case "list": + var pkgs []FWPackage + if err := c.do("GET", "/api/v1/firmware", nil, &pkgs); err != nil { + return err + } + f := "%-44s %-11s %-9s %-22s %9s %s\n" + fmt.Printf(f, "PACKAGE", "VENDOR", "COMPONENT", "VERSION", "SIZE", "FILE / MODELS") + for _, p := range pkgs { + ms := "" + if len(p.Models) > 0 { + ms = " [" + strings.Join(p.Models, ", ") + "]" + } + fmt.Printf(f, p.ID, p.Vendor, p.Component, p.Version, human(float64(p.Size)), p.File+ms) + } + case "rm": + if len(args) < 2 { + usage() + } + return c.do("DELETE", "/api/v1/firmware/"+args[1], nil, nil) + case "show": + if len(args) < 2 { + usage() + } + fs := newFlags("firmware show") + refresh := fs.Bool("refresh", false, "read the inventory from the BMC now") + _ = fs.Parse(args[2:]) + method := "GET" + if *refresh { + method = "POST" + } + var d struct { + Driver string `json:"driver"` + Model string `json:"model"` + Inventory fwInventory `json:"inventory"` + Offers []fwOffer `json:"offers"` + } + if err := c.do(method, "/api/v1/machines/"+args[1]+"/firmware", nil, &d); err != nil { + return err + } + fmt.Printf("%s driver %s model %s inventory %s\n", args[1], orDash(d.Driver), orDash(d.Model), orDash(fwWhen(d.Inventory.At))) + if len(d.Inventory.Components) == 0 { + fmt.Println("no inventory yet: rollcall firmware show " + args[1] + " -refresh") + } + for _, x := range d.Inventory.Components { + up := "" + if !x.Updateable { + up = " (not updateable)" + } + fmt.Printf(" %-9s %-40s %s%s\n", x.Component, trunc(x.Name, 40), x.Version, up) + } + if len(d.Offers) > 0 { + fmt.Println("repository:") + for _, o := range d.Offers { + st := "update" + if o.Same { + st = "installed" + } + fmt.Printf(" %-9s %-22s %-9s (now %s) %s\n", o.Package.Component, o.Package.Version, st, orDash(o.Installed), o.Package.ID) + } + } + case "update": + if len(args) < 3 { + usage() + } + fs := newFlags("firmware update") + apply := fs.String("apply", "", "Immediate | OnReset (default: the BMC's)") + reboot := fs.Bool("reboot", false, "power-cycle the host afterwards so a staged update applies, then verify") + method := fs.String("method", "", "multipart | simple | push (default: the vendor profile's order)") + force := fs.Bool("force", false, "send a package for another vendor or model") + watch := fs.Bool("watch", false, "follow the job") + _ = fs.Parse(args[3:]) + ids, err := c.targets(args[1]) + if err != nil { + return err + } + req := obj{"package": args[2], "apply": *apply, "reboot": *reboot, "method": *method, "force": *force} + for _, id := range ids { + var d obj + if err := c.do("POST", "/api/v1/machines/"+id+"/firmware/update", req, &d); err != nil { + if len(ids) == 1 { + return err + } + fmt.Printf("%s: error: %v\n", id, err) + continue + } + fmt.Printf("%s: job %s %s %s\n", id, str(d["job"]), str(d["component"]), str(d["version"])) + } + if *watch && len(ids) == 1 { + return c.watchJob(ids[0]) + } + default: + usage() + } + return nil +} + +func fwWhen(t time.Time) string { + if t.IsZero() { + return "" + } + return t.Local().Format("2006-01-02 15:04") +} diff --git a/cmd/rollcall/firmware_test.go b/cmd/rollcall/firmware_test.go new file mode 100644 index 0000000..1f8d440 --- /dev/null +++ b/cmd/rollcall/firmware_test.go @@ -0,0 +1,289 @@ +package main + +import ( + "archive/zip" + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// fwBMC is an UpdateService: it takes packages the ways the flags allow and +// keeps a task per upload. +type fwBMC struct { + mu sync.Mutex + multipart, simple, push bool + protocols []string + got []string // method:file-content or method:ImageURI + params []obj + chunked bool + polls int +} + +func (f *fwBMC) handler(t *testing.T) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if u, p, ok := r.BasicAuth(); !ok || u != "admin" || p != "secret" { + w.WriteHeader(401) + return + } + task := func() { + w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/3") + w.WriteHeader(202) + } + switch { + case r.URL.Path == "/redfish/v1/UpdateService": + us := obj{} + if f.multipart { + us["MultipartHttpPushUri"] = "/redfish/v1/UpdateService/upload" + } + if f.push { + us["HttpPushUri"] = "/redfish/v1/UpdateService/push" + } + if f.simple { + act := obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"} + if f.protocols != nil { + act["TransferProtocol@Redfish.AllowableValues"] = f.protocols + } + us["Actions"] = obj{"#UpdateService.SimpleUpdate": act} + } + _ = json.NewEncoder(w).Encode(us) + case r.URL.Path == "/redfish/v1/UpdateService/upload": + f.chunked = f.chunked || r.ContentLength < 0 || len(r.TransferEncoding) > 0 + _, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + w.WriteHeader(400) + return + } + mr := multipart.NewReader(r.Body, ps["boundary"]) + for { + p, err := mr.NextPart() + if err != nil { + break + } + b, _ := io.ReadAll(p) + switch p.FormName() { + case "UpdateParameters": + var o obj + _ = json.Unmarshal(b, &o) + f.params = append(f.params, o) + case "UpdateFile": + f.got = append(f.got, "multipart:"+p.FileName()+":"+string(b)) + } + } + task() + case r.URL.Path == "/redfish/v1/UpdateService/push": + f.chunked = f.chunked || r.ContentLength < 0 + b, _ := io.ReadAll(r.Body) + f.got = append(f.got, "push:"+string(b)) + task() + case r.URL.Path == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate": + var o obj + _ = json.NewDecoder(r.Body).Decode(&o) + f.got = append(f.got, "simple:"+str(o["ImageURI"])+":"+str(o["TransferProtocol"])) + task() + case r.URL.Path == "/redfish/v1/TaskService/Tasks/3": + f.polls++ + st := obj{"TaskState": "Running", "PercentComplete": 40} + if f.polls > 1 { + st = obj{"TaskState": "Completed", "PercentComplete": 100, "Messages": []obj{{"Message": "Firmware update completed"}}} + } + _ = json.NewEncoder(w).Encode(st) + case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory": + _ = json.NewEncoder(w).Encode(obj{"Members": []obj{ + {"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-159-2.19.1"}, + {"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Previous-159-2.18.0"}, + {"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227-7.10.30.00"}, + }}) + case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-159"): + _ = json.NewEncoder(w).Encode(obj{"Id": "Installed-159-2.19.1", "Name": "BIOS", "Version": "2.19.1", "Updateable": true}) + case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227"): + _ = json.NewEncoder(w).Encode(obj{"Id": "Installed-25227-7.10.30.00", "Name": "Integrated Dell Remote Access Controller", "Version": "7.10.30.00", "Updateable": true}) + default: + http.NotFound(w, r) + } + }) +} + +func runFW(t *testing.T, f *fwBMC, op string, arg any) string { + t.Helper() + srv := httptest.NewTLSServer(f.handler(t)) + defer srv.Close() + pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("FIRMWARE-BYTES")) })) + defer pkg.Close() + a := "" + if s, ok := arg.(string); ok { + a = s + } else if arg != nil { + sa := arg.(fwStartArg) + if sa.URL == "" { + sa.URL = pkg.URL + "/firmware/p/BIOS_X.EXE" + } + b, _ := json.Marshal(sa) + a = string(b) + } + out, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), op, a) + if err != nil { + t.Fatalf("%s: %v", op, err) + } + return out +} + +func TestFirmwareMultipart(t *testing.T) { + f := &fwBMC{multipart: true, simple: true} + var res fwStartResult + _ = json.Unmarshal([]byte(runFW(t, f, "fw-start", fwStartArg{File: "BIOS_X.EXE", Apply: "OnReset"})), &res) + if res.Method != "multipart" || res.Task != "/redfish/v1/TaskService/Tasks/3" { + t.Errorf("result %+v", res) + } + if len(f.got) != 1 || f.got[0] != "multipart:BIOS_X.EXE:FIRMWARE-BYTES" || f.chunked { + t.Errorf("got %v chunked %v", f.got, f.chunked) + } + if f.params[0]["@Redfish.OperationApplyTime"] != "OnReset" { + t.Errorf("params %v", f.params) + } +} + +func TestFirmwareSimpleAndPush(t *testing.T) { + f := &fwBMC{simple: true, protocols: []string{"HTTP", "HTTPS", "NFS"}} + _ = runFW(t, f, "fw-start", fwStartArg{File: "x.bin"}) + if len(f.got) != 1 || !strings.HasPrefix(f.got[0], "simple:http://") || !strings.HasSuffix(f.got[0], ":HTTP") { + t.Errorf("simple %v", f.got) + } + g := &fwBMC{push: true} + _ = runFW(t, g, "fw-start", fwStartArg{File: "x.bin"}) + if len(g.got) != 1 || g.got[0] != "push:FIRMWARE-BYTES" || g.chunked { + t.Errorf("push %v chunked %v", g.got, g.chunked) + } + // a service that pulls only over NFS cannot take our HTTP URL: refused clearly + h := &fwBMC{simple: true, protocols: []string{"NFS"}} + srv := httptest.NewTLSServer(h.handler(t)) + defer srv.Close() + b, _ := json.Marshal(fwStartArg{URL: "http://rack/firmware/p/x.bin", File: "x.bin"}) + if _, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), "fw-start", string(b)); err == nil || !strings.Contains(err.Error(), "not HTTP") { + t.Errorf("nfs-only: %v", err) + } +} + +func TestFirmwarePollAndInventory(t *testing.T) { + f := &fwBMC{} + var p fwPollResult + _ = json.Unmarshal([]byte(runFW(t, f, "fw-poll", "/redfish/v1/TaskService/Tasks/3")), &p) + if p.State != "running" || p.Pct != 40 { + t.Errorf("first poll %+v", p) + } + var inv []fwComponent + _ = json.Unmarshal([]byte(runFW(t, f, "fw-inventory", nil)), &inv) + if len(inv) != 2 || inv[0].Component != "bios" || inv[1].Component != "bmc" || inv[0].Version != "2.19.1" { + t.Errorf("inventory %+v", inv) + } + pkgs := []FWPackage{{ID: "a", Vendor: "idrac", Component: "bios", Version: "2.19.1"}, {ID: "b", Vendor: "idrac", Component: "bmc", Version: "7.20.10.50"}} + offers := fwOffers(pkgs, inv) + if !offers[0].Same || offers[1].Same || offers[1].Installed != "7.10.30.00" { + t.Errorf("offers %+v", offers) + } +} + +func TestFirmwareNeedsRedfish(t *testing.T) { + if _, err := powerRun(context.Background(), "", &bmcCreds{Driver: "ipmi"}, "fw-inventory", ""); err == nil || !strings.Contains(err.Error(), "Redfish") { + t.Errorf("ipmi: %v", err) + } +} + +func TestFirmwareRepository(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(t.TempDir(), "BIOS_ABC_WN64_2.19.1.EXE") + _ = os.WriteFile(src, []byte("dup"), 0o644) + p, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "2.19.1", Models: []string{"R650"}}, "") + if err != nil { + t.Fatal(err) + } + if !fwIDRe.MatchString(p.ID) || p.Size != 3 || p.File != "BIOS_ABC_WN64_2.19.1.EXE" { + t.Errorf("package %+v", p) + } + if _, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "x"}, "00"); err == nil { + t.Error("a wrong sha256 was accepted") + } + for _, bad := range []FWPackage{{Vendor: "ipmi", Component: "bios", Version: "1"}, {Vendor: "idrac", Component: "toaster", Version: "1"}, {Vendor: "idrac", Component: "bios"}} { + if _, err := fwImport(context.Background(), dir, src, bad, ""); err == nil { + t.Errorf("accepted %+v", bad) + } + } + list, _ := fwList(dir) + if len(list) != 1 || len(fwFor(list, "idrac", "PowerEdge R650")) != 1 || len(fwFor(list, "idrac", "PowerEdge R760")) != 0 || len(fwFor(list, "ilo", "R650")) != 0 { + t.Errorf("list %+v", list) + } + if _, err := fwGet(dir, "../etc"); err == nil { + t.Error("a path as package id") + } + if err := fwRemove(dir, p.ID); err != nil { + t.Fatal(err) + } + for _, c := range [][2]string{{"2.19.1", "2.19.1"}, {"v2.19.1", "2.19.1"}, {"2.19.1 (Build 4)", "2.19.1"}} { + if !fwSameVersion(c[0], c[1]) { + t.Errorf("%q vs %q", c[0], c[1]) + } + } + if fwSameVersion("2.19.10", "2.19.1") { + t.Error("2.19.10 is not 2.19.1") + } +} + +func TestFirmwareKinds(t *testing.T) { + for in, want := range map[string]string{ + "Installed-159-2.19.1 BIOS": "bios", + "Integrated Dell Remote Access Controller": "bmc", + "PERC H755 Front": "raid", + "RAID Controller": "raid", + "iLO 6": "bmc", + "Broadcom Adv. Dual 25Gb Ethernet": "nic", + "Mellanox ConnectX-6": "nic", + "PSU.Slot.1 Power Supply": "psu", + "System CPLD": "cpld", + "Dell Ent NVMe P5600": "drive", + "Lifecycle Controller": "other", + "OS Collector": "other", + } { + if got := fwKind("", in); got != want { + t.Errorf("%q: %s, want %s", in, got, want) + } + } +} + +func TestFirmwareFwpkg(t *testing.T) { + mk := func(files ...string) string { + p := filepath.Join(t.TempDir(), "cp056789.fwpkg") + f, _ := os.Create(p) + zw := zip.NewWriter(f) + for _, n := range files { + w, _ := zw.Create(n) + _, _ = w.Write([]byte("payload of " + n)) + } + _ = zw.Close() + _ = f.Close() + return p + } + dir := t.TempDir() + got, err := fwImport(context.Background(), dir, mk("payload.json", "ilo6_166.bin"), FWPackage{Vendor: "ilo", Component: "bmc", Version: "1.66"}, "") + if err != nil { + t.Fatal(err) + } + if got.File != "ilo6_166.bin" || got.Size != int64(len("payload of ilo6_166.bin")) || !strings.Contains(got.Notes, "cp056789.fwpkg") { + t.Errorf("extracted %+v", got) + } + if _, err := os.Stat(filepath.Join(dir, got.ID, "ilo6_166.bin")); err != nil { + t.Error(err) + } + if _, err := fwImport(context.Background(), dir, mk("payload.json", "setup.exe"), FWPackage{Vendor: "ilo", Component: "nic", Version: "1"}, ""); err == nil || !strings.Contains(err.Error(), "0 files") { + t.Errorf("agent-only component: %v", err) + } +} diff --git a/cmd/rollcall/fwplans_test.go b/cmd/rollcall/fwplans_test.go new file mode 100644 index 0000000..fa5fb4b --- /dev/null +++ b/cmd/rollcall/fwplans_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// recBMC records what a firmware update sends: every request, the multipart +// parts (JSON parsed), plain form fields and the file's field name. +type recBMC struct { + mu sync.Mutex + vendor string + us obj // the UpdateService resource + inventory []string // FirmwareInventory ids + missing []string // paths that answer 404 + reqs []recReq +} + +type recReq struct { + method, path, ifMatch string + json obj // JSON body, or the UpdateParameters part + parts map[string]obj // JSON parts by name + fields map[string]string + fileField, fileName string + fileBody string +} + +func (f *recBMC) find(method, pathPrefix string) *recReq { + for i := range f.reqs { + if f.reqs[i].method == method && strings.HasPrefix(f.reqs[i].path, pathPrefix) { + return &f.reqs[i] + } + } + return nil +} + +func (f *recBMC) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + for _, m := range f.missing { + if r.URL.Path == m { + http.NotFound(w, r) + return + } + } + if r.URL.Path == "/redfish/v1/SessionService/Sessions" && r.Method == "POST" { + w.Header().Set("X-Auth-Token", "tok") + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/1") + w.WriteHeader(201) + return + } + if r.Method == "DELETE" { + w.WriteHeader(204) + return + } + rq := recReq{method: r.Method, path: r.URL.Path, ifMatch: r.Header.Get("If-Match"), parts: map[string]obj{}, fields: map[string]string{}} + if mt, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type")); err == nil && mt == "multipart/form-data" { + mr := multipart.NewReader(r.Body, ps["boundary"]) + for { + p, err := mr.NextPart() + if err != nil { + break + } + b, _ := io.ReadAll(p) + switch { + case p.FileName() != "": + rq.fileField, rq.fileName, rq.fileBody = p.FormName(), p.FileName(), string(b) + case strings.HasPrefix(p.Header.Get("Content-Type"), "application/json"): + var o obj + _ = json.Unmarshal(b, &o) + rq.parts[p.FormName()] = o + default: + rq.fields[p.FormName()] = string(b) + } + } + rq.json = rq.parts["UpdateParameters"] + } else if r.Body != nil { + _ = json.NewDecoder(r.Body).Decode(&rq.json) + } + if r.Method != "GET" { + f.reqs = append(f.reqs, rq) + } + switch { + case r.URL.Path == "/redfish/v1/": + _ = json.NewEncoder(w).Encode(obj{"Vendor": f.vendor}) + case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "GET": + w.Header().Set("ETag", `"us1"`) + _ = json.NewEncoder(w).Encode(f.us) + case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "PATCH": + for k, v := range rq.json { + f.us[k] = v + } + w.WriteHeader(204) + case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory" && r.Method == "GET": + var ms []obj + for _, id := range f.inventory { + ms = append(ms, obj{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/" + id}) + } + _ = json.NewEncoder(w).Encode(obj{"Members": ms}) + case strings.HasPrefix(r.URL.Path, "/redfish/v1/TaskService/Tasks/"): + _ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100}) + case r.Method == "POST": + w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/9") + w.WriteHeader(202) + default: + http.NotFound(w, r) + } + }) +} + +// fwStartWith runs fw-start for a vendor driver and returns the result and the recorder. +func fwStartWith(t *testing.T, f *recBMC, driver string, a fwStartArg, extra map[string]string) (*fwStartResult, error) { + t.Helper() + srv := httptest.NewTLSServer(f.handler()) + defer srv.Close() + pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("IMG")) })) + defer pkg.Close() + if a.URL == "" { + a.URL = pkg.URL + "/firmware/p/" + a.File + } + c := creds(srv.URL, driver) + for k, v := range extra { + c.Extra[k] = v + } + b, _ := json.Marshal(a) + out, err := powerRun(context.Background(), "", c, "fw-start", string(b)) + if err != nil { + return nil, err + } + var res fwStartResult + _ = json.Unmarshal([]byte(out), &res) + return &res, nil +} + +func mustStart(t *testing.T, f *recBMC, driver string, a fwStartArg) *fwStartResult { + t.Helper() + res, err := fwStartWith(t, f, driver, a, nil) + if err != nil { + t.Fatalf("%s: %v", driver, err) + } + return res +} + +func TestFWPlanDell(t *testing.T) { + f := &recBMC{vendor: "Dell", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/MultipartUpload"}} + mustStart(t, f, "idrac", fwStartArg{File: "BIOS_X_WN64_2.19.1.EXE", Component: "bios"}) + rq := f.find("POST", "/redfish/v1/UpdateService/MultipartUpload") + if rq == nil || rq.json["@Redfish.OperationApplyTime"] != "OnReset" || rq.fileField != "UpdateFile" || rq.fileBody != "IMG" { + t.Errorf("dell %+v", rq) + } +} + +func TestFWPlanSupermicro(t *testing.T) { + f := &recBMC{vendor: "Supermicro", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BIOS", "Motherboard_CPLD_1"}} + mustStart(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"}) + rq := f.find("POST", "/redfish/v1/UpdateService/upload") + if rq == nil || dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveNVRAM") != true || toList(rq.json["Targets"])[0] != "/redfish/v1/Systems/1/Bios" { + t.Errorf("bios params %v", rq.json) + } + f.reqs = nil + mustStart(t, f, "supermicro", fwStartArg{File: "CPLD.bin", Component: "cpld"}) + if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/Motherboard_CPLD_1" { + t.Errorf("cpld params %v", rq.json) + } + // the operator's own parameters (Gen 13 preserve flags) + f.reqs = nil + if _, err := fwStartWith(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"}, map[string]string{"fw_params": `{"Targets":["/redfish/v1/Systems/1/Bios"],"Oem":{"Supermicro":{"BIOS":{"PreserveOA":true}}}}`}); err != nil { + t.Fatal(err) + } + if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveOA") != true { + t.Errorf("fw_params %v", rq.json) + } +} + +func TestFWPlanAMI(t *testing.T) { + f := &recBMC{us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BMCImage1", "BIOS", "BIOS2", "MB_CPLD1"}} + mustStart(t, f, "gigabyte", fwStartArg{File: "rom.ima_enc", Component: "bmc"}) + rq := f.find("POST", "/redfish/v1/UpdateService/upload") + if rq.parts["OemParameters"]["ImageType"] != "BMC" || toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/BMCImage1" { + t.Errorf("gigabyte bmc %+v", rq) + } + f.reqs = nil + mustStart(t, f, "gigabyte", fwStartArg{File: "bios.hpm", Component: "bios"}) + if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); rq.parts["OemParameters"]["ImageType"] != "HPM_BIOS" { + t.Errorf("gigabyte hpm bios %+v", rq.parts) + } + if _, err := fwStartWith(t, &recBMC{us: obj{}, inventory: []string{"BMC"}}, "ami", fwStartArg{File: "x.cap", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "no firmware BIOS") { + t.Errorf("missing target: %v", err) + } +} + +func TestFWPlanXFusion(t *testing.T) { + f := &recBMC{us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}} + mustStart(t, f, "xfusion", fwStartArg{File: "image.hpm", Component: "bmc"}) + up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate") + if up == nil || up.fileField != "file" || len(up.parts) != 0 || su == nil || su.json["ImageURI"] != "/tmp/web/image.hpm" { + t.Errorf("upload %+v simple %+v", up, su) + } + if _, err := fwStartWith(t, f, "xfusion", fwStartArg{File: "nic.hpm", Component: "nic"}, nil); err == nil { + t.Error("in-band NIC firmware accepted") + } +} + +func TestFWPlanKaytus(t *testing.T) { + f := &recBMC{us: obj{"ServiceEnabled": true, "Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}} + res := mustStart(t, f, "kaytus", fwStartArg{File: "bios.bin", Component: "bios"}) + up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions") + if up.fileField != "fwimage" || up.fields["CurrentPassword"] != "secret" || up.fields["EncryptFlag"] != "False" { + t.Errorf("upload %+v", up) + } + if dig(su.json, "Oem", "Public", "FlashItem") != "BIOS" || dig(su.json, "Oem", "Public", "BiosFlash") != "Both" { + t.Errorf("flash %v", su.json) + } + if res.Task == "" { + t.Errorf("no task") + } +} + +func TestFWPlanIRMC(t *testing.T) { + f := &recBMC{vendor: "ts_fujitsu", us: obj{}, missing: []string{"/redfish/v1/Managers/iRMC/Actions/FTSManager.FWUpdate"}} + mustStart(t, f, "irmc", fwStartArg{File: "D3279_09.09F.bin", Component: "bmc"}) + rq := f.find("POST", "/redfish/v1/Managers/iRMC/Actions/Oem/FTSManager.FWUpdate") + if rq == nil || rq.fileField != "data" || rq.json != nil { + t.Errorf("irmc %+v (all %+v)", rq, f.reqs) + } + g := &recBMC{vendor: "Fsas", us: obj{}} + mustStart(t, g, "irmc", fwStartArg{File: "D3279-B1x.R1.20.0.UPC", Component: "bios"}) + if g.find("POST", "/redfish/v1/Systems/0/Bios/Actions/Oem/FsasBios.BiosUpdate") == nil { + t.Errorf("fsas bios %+v", g.reqs) + } +} + +func TestFWPlanLenovoClaims(t *testing.T) { + f := &recBMC{vendor: "Lenovo", us: obj{"MultipartHttpPushUri": "/mfwupdate", "HttpPushUriTargetsBusy": false}} + mustStart(t, f, "xcc", fwStartArg{File: "lnvgy_fw_xcc.uxz", Component: "bmc"}) + if f.us["HttpPushUriTargetsBusy"] != true || f.find("POST", "/mfwupdate") == nil || f.find("PATCH", "/redfish/v1/UpdateService").ifMatch != `"us1"` { + t.Errorf("claim %v %+v", f.us, f.reqs) + } + if _, err := fwStartWith(t, f, "xcc", fwStartArg{File: "x.uxz", Component: "bmc"}, nil); err == nil || !strings.Contains(err.Error(), "busy") { + t.Errorf("second upload while busy: %v", err) + } + srv := httptest.NewTLSServer(f.handler()) + defer srv.Close() + if _, err := powerRun(context.Background(), "", creds(srv.URL, "xcc"), "fw-finish", ""); err != nil || f.us["HttpPushUriTargetsBusy"] != false { + t.Errorf("release: %v %v", err, f.us) + } +} + +func TestFWPlanILOAndCisco(t *testing.T) { + f := &recBMC{vendor: "HPE", us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}} + mustStart(t, f, "ilo", fwStartArg{File: "ilo6_166.bin", Component: "bmc"}) + if rq := f.find("POST", "/redfish/v1/UpdateService/Actions"); rq == nil || !strings.HasSuffix(str(rq.json["ImageURI"]), "/ilo6_166.bin") { + t.Errorf("ilo %+v", f.reqs) + } + if _, err := fwStartWith(t, &recBMC{us: obj{}}, "cimc", fwStartArg{File: "x.bin", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "SCP") { + t.Errorf("cimc: %v", err) + } +} + +func TestFWPollStaged(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/redfish/v1/TaskService/Tasks/JID_1": + _ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 100, "Messages": []obj{{"Message": "Job scheduled successfully."}}}) + case "/redfish/v1/TaskService/Tasks/BIOS0": + _ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100}) + case "/redfish/v1/TaskService/Tasks/BIOS1": + _ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 30}) + case "/redfish/v1/JobService/Jobs/JID_2": + _ = json.NewEncoder(w).Encode(obj{"JobState": "Completed", "PercentComplete": 100}) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + poll := func(task string) fwPollResult { + out, err := powerRun(context.Background(), "", creds(srv.URL, "idrac"), "fw-poll", task) + if err != nil { + t.Fatal(err) + } + var p fwPollResult + _ = json.Unmarshal([]byte(out), &p) + return p + } + if p := poll("/redfish/v1/TaskService/Tasks/JID_1"); p.State != "done" { + t.Errorf("scheduled job: %+v", p) + } + if p := poll("/redfish/v1/TaskService/Tasks/BIOS0,/redfish/v1/TaskService/Tasks/BIOS1"); p.State != "running" || p.Pct != 30 { + t.Errorf("second of two: %+v", p) + } + if p := poll("/redfish/v1/TaskService/Tasks/JID_2"); p.State != "done" { // moved to the job service + t.Errorf("job service: %+v", p) + } + if p := poll("/redfish/v1/TaskService/Tasks/GONE"); p.State != "unknown" { + t.Errorf("gone: %+v", p) + } +} diff --git a/cmd/rollcall/ipmi.go b/cmd/rollcall/ipmi.go index 811ac84..803b199 100644 --- a/cmd/rollcall/ipmi.go +++ b/cmd/rollcall/ipmi.go @@ -11,6 +11,7 @@ import ( "log" "net/http" "strings" + "time" ) // bmcCreds is what a power driver needs. @@ -83,6 +84,24 @@ func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg st Password: c.Password, Extra: c.Extra, Op: op, Arg: arg}) } +// firmwareVia runs a firmware operation (fw-*) where the BMC is reachable. +func (a *App) firmwareVia(ctx context.Context, m *Machine, op, arg string) (string, error) { + c, err := a.credsFor(m) + if err != nil { + return "", fmt.Errorf("%w: %v", errNoBMC, err) + } + rack := a.rackOf(m) + if rack == a.cfg.RackName || rack == "" { + return powerRun(ctx, a.cfg.IPMITool, c, op, arg) + } + r, ok := a.racksSnapshot()[rack] + if !ok || !rackOnline(r, a.cfg.RackName) { + return "", fmt.Errorf("rack controller %s is offline", rack) + } + return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User, + Password: c.Password, Extra: c.Extra, Op: op, Arg: arg, Timeout: int((fwTimeouts[op] + time.Minute).Seconds())}) +} + func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error { if secret != nil { _, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret) diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index bca9302..02f5125 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -49,6 +49,8 @@ func usage() { rollcall unassign ID rollcall overlay ID [-stage system|runtime] [-extract DIR] rollcall render PROFILE PATH [-stage system|runtime] + rollcall firmware list | import FILE|URL -vendor V -component C -version X [-models M,..] [-sha256 S] | rm PKG + rollcall firmware show MACHINE [-refresh] | update MACHINE|tag:T PKG [-apply OnReset|Immediate] [-reboot] [-method M] [-force] [-watch] rollcall os list | sync [NAME...] [-force] | import NAME FILE [-family F] [-version V] | promote NAME VERSION | rm NAME VERSION rollcall deploy ID OS [-version V] [-disk /dev/sdX] [-hostname H] [-upgrade] [-user-data FILE] [-boot] [-watch] rollcall release ID [-boot] [-watch] @@ -147,6 +149,11 @@ func main() { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } + case "firmware": + if err := firmwareCmd(args); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } case "os": if err := osCmd(args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) diff --git a/cmd/rollcall/power.go b/cmd/rollcall/power.go index 7817b9b..e911ef9 100644 --- a/cmd/rollcall/power.go +++ b/cmd/rollcall/power.go @@ -57,6 +57,11 @@ var errManualPower = errors.New("this machine has manual power control: do it by // powerRun executes one operation with a driver. func powerRun(ctx context.Context, ipmitool string, c *bmcCreds, op, arg string) (string, error) { + if t, ok := fwTimeouts[op]; ok { // firmware: Redfish UpdateService (redfish_fw.go) + ctx, cancel := context.WithTimeout(ctx, t) + defer cancel() + return firmwareRun(ctx, c, op, arg) + } if !powerOps[op] { return "", fmt.Errorf("unknown power operation %q", op) } diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index ad027a1..b211943 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -76,6 +76,7 @@ type rackJob struct { Extra map[string]string `json:"extra"` Op string `json:"op"` Arg string `json:"arg"` + Timeout int `json:"timeout,omitempty"` // seconds the region waits (default 90) done chan rackResult rack string // who took it: only that rack may answer } @@ -242,7 +243,11 @@ func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string delete(h.wake, rack) } h.mu.Unlock() - t := time.NewTimer(90 * time.Second) + wait := 90 * time.Second + if j.Timeout > 0 { + wait = time.Duration(j.Timeout) * time.Second + } + t := time.NewTimer(wait) defer t.Stop() select { case r := <-j.done: diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 19beb37..5f61bd8 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -358,7 +358,7 @@ func runRack(args []string) error { } go r.loop() mux := http.NewServeMux() - for _, p := range []string{"/rollcall/", "/images/", "/os/"} { + for _, p := range []string{"/rollcall/", "/images/", "/os/", "/firmware/"} { // firmware: BMCs pull packages from here mux.HandleFunc("GET "+p, r.serveCached) mux.HandleFunc("HEAD "+p, r.serveCached) } @@ -546,8 +546,8 @@ func (r *rackd) head(rel string) (headInfo, error) { // fresh says whether the cached copy matches the region's file. OS images live // under versioned paths and never change; the rest is checked by size and time. func (r *rackd) fresh(rel string, fi os.FileInfo) bool { - if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") { - return true + if (strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/")) || strings.HasPrefix(rel, "/firmware/") { + return true // packages live under their id and never change } h, err := r.head(rel) if err != nil { diff --git a/cmd/rollcall/redfish.go b/cmd/rollcall/redfish.go index 704b8b7..2c2507d 100644 --- a/cmd/rollcall/redfish.go +++ b/cmd/rollcall/redfish.go @@ -49,8 +49,20 @@ type redfishProfile struct { // Clear: bodies that clear a boot override, tried in order (default: Enabled Disabled) Clear []obj // Auth: "session" logs in through the SessionService (X-Auth-Token); default basic. - Auth string - Sources []string + Auth string + // Firmware (UpdateService, redfish_fw.go). FWPlan builds the vendor's + // documented request for a component; without one the standard methods are + // tried in FWMethods order (default multipart, simple, push). FWUnsupported + // says why Rollcall cannot update this vendor's firmware. FWFormats are the + // package files it takes, checked at import. + FWPlan func(fwPlanCtx) (*fwRequest, error) + FWMethods []string + FWUnsupported string + FWClaims bool // the plan takes HttpPushUriTargetsBusy; fw-finish gives it back + FWFormats []string + FWNotes string + FWSources []string + Sources []string } // redfishProfiles, most specific first: the AMI-based ones (Gigabyte, ASRock @@ -168,6 +180,7 @@ type rfClient struct { hc *http.Client etags map[string]string // resource -> ETag from its last GET token, session string // X-Auth-Token and the session to delete, when logged in + pushURIs map[string]bool // upload URIs the UpdateService advertised (Lenovo: /mfwupdate) } // login opens a session (profile or extra auth=session); logout closes it. @@ -228,6 +241,9 @@ func (r *rfClient) resolve(p string) (string, error) { } p = u.RequestURI() } + if r.pushURIs[p] && strings.HasPrefix(p, "/") && !strings.Contains(p, "..") { + return p, nil // the BMC's own upload URI, on this BMC (outside /redfish/ on some) + } if !strings.HasPrefix(p, "/redfish/") || strings.Contains(p, "/../") || strings.HasSuffix(p, "/..") { return "", fmt.Errorf("the BMC pointed at %q, outside /redfish/", p) } diff --git a/cmd/rollcall/redfish_fw.go b/cmd/rollcall/redfish_fw.go new file mode 100644 index 0000000..b86f9f0 --- /dev/null +++ b/cmd/rollcall/redfish_fw.go @@ -0,0 +1,666 @@ +package main + +// Firmware over Redfish (UpdateService), run where the BMC is reachable (the +// rack controller). Three operations, each short enough for one rack command: +// +// fw-inventory what is installed (FirmwareInventory) +// fw-start hand a package to the BMC; returns the task to watch +// fw-poll one look at that task +// +// The package reaches the BMC one of three ways, chosen from what the +// UpdateService offers and the vendor profile's order: +// +// multipart POST MultipartHttpPushUri: UpdateParameters (JSON) + UpdateFile +// simple UpdateService.SimpleUpdate with an ImageURI the BMC downloads +// from the rack controller (or firmware_base) +// push POST HttpPushUri with the file as the body +// +// The rack controller fetches the package from its own cache (filled from the +// region) and sends it with a known length: many BMC web servers refuse +// chunked uploads. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// fwComponent is one entry of a machine's firmware inventory. +type fwComponent struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` + Component string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other + Updateable bool `json:"updateable"` + SoftwareID string `json:"software_id,omitempty"` +} + +// fwStartArg is what fw-start receives. +type fwStartArg struct { + URL string `json:"url"` // where the package is (the BMC pulls it, or the rack downloads it to push) + Component string `json:"component"` // bios | bmc | ... (vendor plans pick targets by it) + File string `json:"file"` // its file name + Method string `json:"method,omitempty"` // multipart | simple | push; "" = profile order + Apply string `json:"apply,omitempty"` // Immediate | OnReset; "" = the BMC's default + Targets []string `json:"targets,omitempty"` // inventory resources to update; usually empty +} + +type fwStartResult struct { + Method string `json:"method"` + Task string `json:"task,omitempty"` // task monitor or task resource; "" when the BMC answered synchronously + Note string `json:"note,omitempty"` +} + +// fwTimeouts bound each firmware operation; an upload to a slow BMC takes long. +var fwTimeouts = map[string]time.Duration{"fw-inventory": 5 * time.Minute, "fw-start": 45 * time.Minute, "fw-poll": 2 * time.Minute, "fw-finish": time.Minute} + +type fwPollResult struct { + State string `json:"state"` // running | done | failed | unknown + Pct int `json:"pct"` + Msg string `json:"msg,omitempty"` +} + +// fwKinds sorts inventory entries by id and name, most specific first: a RAID +// "controller" is not the management controller. +var fwKinds = []struct { + kind string + re *regexp.Regexp +}{ + {"bios", regexp.MustCompile(`(?i)\b(bios|uefi|system rom)\b`)}, + {"raid", regexp.MustCompile(`(?i)\b(raid|perc|smart ?array|megaraid|hba|storage controller|boss)\b`)}, + {"nic", regexp.MustCompile(`(?i)\b(nic|ethernet|network|connectx|mellanox|infiniband|e810|x710|bcm\d*|broadcom adv)`)}, + {"psu", regexp.MustCompile(`(?i)\b(psu|power supply)\b`)}, + {"cpld", regexp.MustCompile(`(?i)\b(cpld|fpga)\b`)}, + {"drive", regexp.MustCompile(`(?i)\b(drive|disk|ssd|nvme|hdd)\b`)}, + {"bmc", regexp.MustCompile(`(?i)\b(bmc|idrac|ilo ?\d?|xcc2?|imm2?|ibmc|irmc|cimc|remote access controller|management controller|baseboard management)\b`)}, +} + +// fwKind sorts an inventory entry into a component class by its id and name. +func fwKind(id, name string) string { + s := id + " " + name + for _, k := range fwKinds { + if k.re.MatchString(s) { + return k.kind + } + } + return "other" +} + +func firmwareRun(ctx context.Context, c *bmcCreds, op, arg string) (string, error) { + d, _ := driverByName(c.Driver) + if d.Family != "redfish" { + return "", fmt.Errorf("firmware updates need a Redfish driver (this BMC uses %s)", orDefault(c.Driver, "ipmi")) + } + prof := redfishProfileFor(c.Driver) + r := newRF(c) + r.hc.Timeout = 0 // uploads take minutes; the context bounds each call + if err := r.open(ctx, c, prof); err != nil { + return "", err + } + defer r.logout() + var out any + var err error + switch op { + case "fw-inventory": + out, err = r.fwInventory(ctx) + case "fw-start": + var a fwStartArg + if err := json.Unmarshal([]byte(arg), &a); err != nil { + return "", fmt.Errorf("fw-start: %w", err) + } + out, err = r.fwStart(ctx, c, prof, a) + case "fw-poll": + out, err = r.fwPoll(ctx, arg) + case "fw-finish": // after the task ended: give back what fw-start claimed + if prof.FWClaims { + r.fwRelease(ctx) + } + out = obj{"ok": true} + default: + return "", fmt.Errorf("unknown firmware operation %q", op) + } + if err != nil { + return "", err + } + b, _ := json.Marshal(out) + return string(b), nil +} + +func (r *rfClient) fwInventory(ctx context.Context) ([]fwComponent, error) { + var coll struct { + Members []struct { + ID string `json:"@odata.id"` + } `json:"Members"` + } + if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err != nil { + return nil, err + } + out := []fwComponent{} + for i, mem := range coll.Members { + if i >= 300 { + break + } + id := path.Base(strings.TrimRight(mem.ID, "/")) + // Dell lists what is installed and what was before or is staged + if strings.HasPrefix(id, "Previous-") || strings.HasPrefix(id, "Available-") { + continue + } + var e obj + if _, err := r.do(ctx, "GET", mem.ID, nil, &e); err != nil { + continue + } + up, _ := e["Updateable"].(bool) + fc := fwComponent{ID: str(e["Id"]), Name: str(e["Name"]), Version: str(e["Version"]), Updateable: up, SoftwareID: str(e["SoftwareId"])} + if fc.ID == "" { + fc.ID = id + } + fc.Component = fwKind(fc.ID, fc.Name) + out = append(out, fc) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Component != out[j].Component { + return out[i].Component < out[j].Component + } + return out[i].ID < out[j].ID + }) + return out, nil +} + +// fwCaps is what an UpdateService offers. +type fwCaps struct { + simple, multipart, push string + protocols []string +} + +func (r *rfClient) fwCaps(ctx context.Context) (fwCaps, error) { + var us obj + if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil { + return fwCaps{}, err + } + act := asObj(dig(us, "Actions", "#UpdateService.SimpleUpdate")) + if r.pushURIs == nil { + r.pushURIs = map[string]bool{} + } + for _, k := range []string{"MultipartHttpPushUri", "HttpPushUri"} { + if u := str(us[k]); u != "" && !strings.Contains(u, "://") { + r.pushURIs[u] = true + } + } + return fwCaps{ + simple: str(act["target"]), + protocols: strList(act["TransferProtocol@Redfish.AllowableValues"]), + multipart: str(us["MultipartHttpPushUri"]), + push: str(us["HttpPushUri"]), + }, nil +} + +// fwRequest is one way to hand a package over, as a vendor documents it. +type fwRequest struct { + Method string // multipart | simple | push: how this request carries the package + URI string // where it goes + AltURI string // tried when URI answers 404 (paths that moved between firmware generations) + Params obj // multipart UpdateParameters, or the JSON body of a simple request + NoParams bool // multipart without an UpdateParameters part + Parts []fwPart // more parts before the file (AMI OemParameters) + Field string // the file's form field (default UpdateFile) + Form map[string]string // text fields after the file + FailText string // a 2xx answer that contains this is a refusal (KAYTUS "Verification Failed") + Claim bool // take HttpPushUriTargetsBusy first (Lenovo); fw-finish gives it back + Then *fwRequest // a second call naming the uploaded file (xFusion, KAYTUS) + Tasks []string // fixed tasks to watch when the answers name none (KAYTUS) +} + +type fwPart struct { + Name string + Body obj +} + +// fwPlanCtx is what a vendor plan decides from. +type fwPlanCtx struct { + Component, File, URL, Apply, Password string + Vendor string // ServiceRoot Vendor + Caps fwCaps // what the UpdateService offers + Inventory []string // FirmwareInventory member ids +} + +func (x fwPlanCtx) has(id string) bool { return hasTag(x.Inventory, id) } + +func (x fwPlanCtx) first(ids ...string) string { + for _, id := range ids { + if x.has(id) { + return id + } + } + return "" +} + +func (x fwPlanCtx) applyOr(def string) string { return orDefault(x.Apply, def) } + +func (r *rfClient) fwStart(ctx context.Context, c *bmcCreds, prof redfishProfile, a fwStartArg) (*fwStartResult, error) { + caps, err := r.fwCaps(ctx) + if err != nil { + return nil, fmt.Errorf("no UpdateService: %w", err) + } + forced := orDefault(a.Method, c.Extra["fw_method"]) + if prof.FWUnsupported != "" && forced == "" { + return nil, errors.New(prof.FWUnsupported + " (to try a standard method anyway: -method multipart|simple|push)") + } + if prof.FWPlan != nil && forced == "" { + x := fwPlanCtx{Component: a.Component, File: a.File, URL: a.URL, Apply: a.Apply, Password: r.pass, Caps: caps} + var root obj + if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err == nil { + x.Vendor = str(root["Vendor"]) + } + var coll struct { + Members []struct { + ID string `json:"@odata.id"` + } `json:"Members"` + } + if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err == nil { + for _, m := range coll.Members { + x.Inventory = append(x.Inventory, path.Base(strings.TrimRight(m.ID, "/"))) + } + } + req, err := prof.FWPlan(x) + if err != nil { + return nil, err + } + if raw := c.Extra["fw_params"]; raw != "" { // the operator's own UpdateParameters / body + var o obj + if err := json.Unmarshal([]byte(raw), &o); err != nil { + return nil, fmt.Errorf("fw_params is not JSON: %w", err) + } + req.Params = o + } + return r.fwExec(ctx, req, a) + } + order := prof.FWMethods + if len(order) == 0 { + order = []string{"multipart", "simple", "push"} + } + if forced != "" { + order = []string{forced} + } + var tried []string + for _, m := range order { + req := fwStandard(caps, m, a) + if req == nil { + continue + } + res, err := r.fwExec(ctx, req, a) + if err == nil { + return res, nil + } + // a BMC that does not like this way gets the next one; a refused + // package (wrong model, signature, downgrade) stops here + if e := asRFError(err); e == nil || (e.code != http.StatusMethodNotAllowed && e.code != http.StatusNotImplemented && e.code != http.StatusUnsupportedMediaType) { + return nil, fmt.Errorf("%s: %w", m, err) + } + tried = append(tried, m+": "+err.Error()) + } + if len(tried) > 0 { + return nil, fmt.Errorf("the BMC took no upload method: %s", strings.Join(tried, "; ")) + } + return nil, fmt.Errorf("the BMC's UpdateService offers none of %v", order) +} + +// fwStandard is the DMTF request for a method, nil when the service lacks it. +func fwStandard(caps fwCaps, method string, a fwStartArg) *fwRequest { + switch method { + case "multipart": + if caps.multipart == "" { + return nil + } + params := obj{"Targets": orEmpty(a.Targets)} + if a.Apply != "" { + params["@Redfish.OperationApplyTime"] = a.Apply + } + return &fwRequest{Method: "multipart", URI: caps.multipart, Params: params} + case "simple": + if caps.simple == "" { + return nil + } + body := obj{"ImageURI": a.URL} + if len(caps.protocols) > 0 { // only when the service lists them: others want the scheme in the URI + body["TransferProtocol"] = strings.ToUpper(strings.SplitN(a.URL, "://", 2)[0]) + } + if len(a.Targets) > 0 { + body["Targets"] = a.Targets + } + return &fwRequest{Method: "simple", URI: caps.simple, Params: body} + case "push": + if caps.push == "" { + return nil + } + return &fwRequest{Method: "push", URI: caps.push} + } + return nil +} + +// fwExec carries out a request (and its follow-up) and names the task to watch. +func (r *rfClient) fwExec(ctx context.Context, req *fwRequest, a fwStartArg) (*fwStartResult, error) { + if req.Claim { + var us obj + if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil { + return nil, err + } + if us["HttpPushUriTargetsBusy"] == true { + return nil, errors.New("the BMC's update service is busy with another upload (HttpPushUriTargetsBusy)") + } + if _, err := r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": true}); err != nil { + return nil, fmt.Errorf("claim the update service: %w", err) + } + } + resp, err := r.fwSend(ctx, req, a) + if err != nil { + if req.Claim { + r.fwRelease(ctx) + } + return nil, err + } + res := &fwStartResult{Method: req.Method, Task: fwTaskOf(resp)} + if req.Then != nil { + tresp, err := r.fwSend(ctx, req.Then, a) + if err != nil { + return nil, fmt.Errorf("the BMC has the file, but did not start the update: %w", err) + } + res.Task = fwTaskOf(tresp) + } + if res.Task == "" && len(req.Tasks) > 0 { + res.Task = strings.Join(req.Tasks, ",") + } + return res, nil +} + +func (r *rfClient) fwRelease(ctx context.Context) { + _, _ = r.call(ctx, "GET", "/redfish/v1/UpdateService", nil, "") + _, _ = r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": false}) +} + +func (r *rfClient) fwSend(ctx context.Context, req *fwRequest, a fwStartArg) (*rfResp, error) { + var resp *rfResp + var err error + send := func(uri string) (*rfResp, error) { + switch req.Method { + case "simple": + if proto := str(req.Params["TransferProtocol"]); proto != "" && len(a.URL) > 0 { + // checked here so a service that pulls only over NFS/SCP says so clearly + if caps, cerr := r.fwCaps(ctx); cerr == nil && len(caps.protocols) > 0 && !hasTag(caps.protocols, proto) { + return nil, &rfError{method: "POST", path: uri, code: http.StatusNotImplemented, msg: fmt.Sprintf("transfer protocols %v, not %s", caps.protocols, proto)} + } + } + return r.call(ctx, "POST", uri, req.Params, "") + case "multipart", "push": + f, size, ferr := fwFetch(ctx, a.URL) + if ferr != nil { + return nil, ferr + } + defer os.Remove(f.Name()) + defer f.Close() + if req.Method == "push" { + return r.upload(ctx, uri, "application/octet-stream", f, size, r.etags[uri]) + } + return r.multipartUpload(ctx, uri, req, a.File, f, size) + } + return nil, fmt.Errorf("unknown method %q", req.Method) + } + resp, err = send(req.URI) + if e := asRFError(err); e != nil && e.code == http.StatusNotFound && req.AltURI != "" { + resp, err = send(req.AltURI) + } + if err == nil && req.FailText != "" && strings.Contains(string(resp.body), req.FailText) { + return nil, fmt.Errorf("the BMC refused the package: %s", req.FailText) + } + return resp, err +} + +func orEmpty(s []string) []string { + if s == nil { + return []string{} + } + return s +} + +// fwTaskOf finds the task in an answer: Location, a Task body, or a Dell job id. +func fwTaskOf(resp *rfResp) string { + if resp == nil { + return "" + } + if l := resp.hdr.Get("Location"); l != "" { + return l + } + var t struct { + ID string `json:"@odata.id"` + Type string `json:"@odata.type"` + } + if json.Unmarshal(resp.body, &t) == nil && strings.Contains(t.Type, "Task") { + return t.ID + } + return "" +} + +// fwFetch downloads the package to a temporary file (the rack's cache serves it). +func fwFetch(ctx context.Context, url string) (*os.File, int64, error) { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, 0, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("package: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, 0, fmt.Errorf("package %s: %s", url, resp.Status) + } + f, err := os.CreateTemp("", "rollcall-fw-*") + if err != nil { + return nil, 0, err + } + n, err := io.Copy(f, resp.Body) + if err == nil { + _, err = f.Seek(0, io.SeekStart) + } + if err != nil { + f.Close() + os.Remove(f.Name()) + return nil, 0, err + } + return f, n, nil +} + +// upload POSTs a body of known length (no chunked encoding). +func (r *rfClient) upload(ctx context.Context, target, ctype string, body io.Reader, size int64, ifMatch string) (*rfResp, error) { + p, err := r.resolve(target) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, "POST", r.base+p, body) + if err != nil { + return nil, err + } + req.ContentLength = size + req.Header.Set("Content-Type", ctype) + req.Header.Set("Accept", "application/json") + if ifMatch != "" { + req.Header.Set("If-Match", ifMatch) + } + if r.token != "" { + req.Header.Set("X-Auth-Token", r.token) + } else if r.user != "" { + req.SetBasicAuth(r.user, r.pass) + } + resp, err := r.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + out := &rfResp{code: resp.StatusCode, hdr: resp.Header, body: data} + if resp.StatusCode >= 300 { + ids, msg := redfishMessage(data) + return out, &rfError{method: "POST", path: p, code: resp.StatusCode, ids: ids, msg: resp.Status + " " + msg} + } + return out, nil +} + +// multipartUpload sends UpdateParameters, any OEM parts, the file and text +// fields (DSP0266 "multipart HTTP push", plus what vendors add to it). +func (r *rfClient) multipartUpload(ctx context.Context, target string, req *fwRequest, name string, f io.Reader, size int64) (*rfResp, error) { + var head, after bytes.Buffer + mw := multipart.NewWriter(&head) + jsonPart := func(n string, v any) { + h := textproto.MIMEHeader{} + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q`, n)) + h.Set("Content-Type", "application/json") + w, _ := mw.CreatePart(h) + b, _ := json.Marshal(v) + _, _ = w.Write(b) + } + if !req.NoParams { + p := req.Params + if p == nil { + p = obj{"Targets": []string{}} + } + jsonPart("UpdateParameters", p) + } + for _, p := range req.Parts { + jsonPart(p.Name, p.Body) + } + fh := textproto.MIMEHeader{} + fh.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, orDefault(req.Field, "UpdateFile"), path.Base(name))) + fh.Set("Content-Type", "application/octet-stream") + _, _ = mw.CreatePart(fh) + // text fields and the closing boundary come after the file + keys := make([]string, 0, len(req.Form)) + for k := range req.Form { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(&after, "\r\n--%s\r\nContent-Disposition: form-data; name=%q\r\n\r\n%s", mw.Boundary(), k, req.Form[k]) + } + fmt.Fprintf(&after, "\r\n--%s--\r\n", mw.Boundary()) + body := io.MultiReader(&head, f, &after) + return r.upload(ctx, target, mw.FormDataContentType(), body, int64(head.Len())+size+int64(after.Len()), r.etags[target]) +} + +// fwPoll looks at the task(s) of an update; several (comma-separated) are +// watched in order and count as done when all are. +func (r *rfClient) fwPoll(ctx context.Context, tasks string) (*fwPollResult, error) { + if tasks == "" { + return &fwPollResult{State: "unknown"}, nil + } + var last *fwPollResult + seen := false + for _, t := range strings.Split(tasks, ",") { + res, err := r.fwPollOne(ctx, t) + if err != nil { + return nil, err + } + if res.State == "unknown" { // not (or no longer) listed: the others decide + continue + } + seen = true + if res.State != "done" { + return res, nil + } + last = res + } + if !seen { + return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil + } + return last, nil +} + +var fwScheduledRe = regexp.MustCompile(`(?i)\bscheduled\b|completed successfully`) + +func (r *rfClient) fwPollOne(ctx context.Context, task string) (*fwPollResult, error) { + resp, err := r.call(ctx, "GET", task, nil, "") + if e := asRFError(err); e != nil && e.code == http.StatusNotFound { + // iDRAC: while it updates itself the task moves to the job service + if i := strings.Index(task, "/TaskService/Tasks/"); i >= 0 { + resp, err = r.call(ctx, "GET", "/redfish/v1/JobService/Jobs/"+path.Base(task), nil, "") + } + if e := asRFError(err); e != nil && e.code == http.StatusNotFound { + return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil + } + } + if err != nil { + return nil, err + } + var t obj + _ = json.Unmarshal(resp.body, &t) + st := str(t["TaskState"]) + str(t["JobState"]) + res := &fwPollResult{State: "running", Pct: -1} + if p, ok := t["PercentComplete"].(float64); ok { + res.Pct = int(p) + } + for _, v := range asObj(t["Oem"]) { // iBMC TaskPercentage "40%", iRMC TotalProgressPercent + o := asObj(v) + if p := strings.TrimSuffix(str(o["TaskPercentage"]), "%"); p != "" { + res.Pct, _ = strconv.Atoi(p) + } + if p, ok := o["TotalProgressPercent"].(float64); ok { + res.Pct = int(p) + } + } + if ms := toList(t["Messages"]); len(ms) > 0 { + res.Msg = str(asObj(ms[len(ms)-1])["Message"]) + } + if res.Msg == "" { + res.Msg = str(t["Message"]) // Dell jobs + } + if res.Pct < 0 { // no percentage on the task: some keep it on the update service + res.Pct = 0 + var us obj + if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err == nil { + if h := asObj(dig(us, "Oem", "Hpe")); len(h) > 0 { // iLO: State, FlashProgressPercent, Result + if p, ok := h["FlashProgressPercent"].(float64); ok { + res.Pct = int(p) + } + if str(h["State"]) == "Error" { + res.State, res.Msg = "failed", "iLO update service: "+oneLine(fmt.Sprint(h["Result"])) + return res, nil + } + } + if a := asObj(dig(us, "Oem", "AMIUpdateService")); len(a) > 0 { // "3% done." + if m := regexp.MustCompile(`(\d+)%`).FindStringSubmatch(str(a["FlashPercentage"])); m != nil { + res.Pct, _ = strconv.Atoi(m[1]) + } + if res.Msg == "" { + res.Msg = str(a["UpdateStatus"]) + } + } + } + } + switch st { + case "Completed": + res.State, res.Pct = "done", 100 + case "Exception", "Killed", "Cancelled", "Failed", "CompletedWithErrors", "Interrupted": + res.State = "failed" + case "Scheduled", "UserIntervention": // Dell: staged until the host reboots + res.State = "done" + case "": + if resp.code == http.StatusOK && t["TaskState"] == nil && t["JobState"] == nil { + res.State = "done" // a task monitor answers with the result once finished + } + } + if res.State == "running" && res.Pct >= 100 && fwScheduledRe.MatchString(res.Msg) { + res.State = "done" // staged for the next reboot + } + return res, nil +} diff --git a/cmd/rollcall/redfish_fwplans.go b/cmd/rollcall/redfish_fwplans.go new file mode 100644 index 0000000..2e87406 --- /dev/null +++ b/cmd/rollcall/redfish_fwplans.go @@ -0,0 +1,273 @@ +package main + +// Firmware update plans per vendor: the request each one documents for a +// component. Sources are on each entry; "oss" ones come from the vendor's own +// tools (Ansible collections, PowerShell modules, SDKs) where its API guide is +// not public. An operator can replace a plan's UpdateParameters with +// fw_params={...} on the BMC record, or skip the plan with fw_method=. + +import ( + "errors" + "fmt" + "path" + "strings" +) + +type fwVendor struct { + plan func(fwPlanCtx) (*fwRequest, error) + methods []string + claims bool + unsupported string + formats []string + notes string + sources []string +} + +var fwVendors = map[string]fwVendor{ + "idrac": { + // MultipartUpload; older iDRAC 9 without it pulls with SimpleUpdate. + // OnReset by default: BIOS-type jobs wait for a reboot anyway, and an + // iDRAC or CPLD update must not surprise anyone. + plan: func(x fwPlanCtx) (*fwRequest, error) { + apply := x.applyOr("OnReset") + if x.Caps.multipart != "" { + return &fwRequest{Method: "multipart", URI: x.Caps.multipart, + Params: obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply, "Oem": obj{}}}, nil + } + if x.Caps.simple != "" { + return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: simpleBody(x, obj{"@Redfish.OperationApplyTime": apply})}, nil + } + return nil, errors.New("this iDRAC offers neither MultipartHttpPushUri nor SimpleUpdate") + }, + formats: []string{".exe", ".d9", ".d7", ".pm"}, + notes: "Dell Update Packages (Windows .EXE). BIOS, RAID, NIC jobs apply at the next reboot (-reboot); iDRAC, Diags apply by themselves; a CPLD update does a virtual A/C cycle.", + sources: []string{ + "https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareMultipartUploadREDFISH.py", + "https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareSimpleUpdateTransferProtocolREDFISH.py", + }, + }, + "ilo": { + // SimpleUpdate over HTTP(S) flashes iLO .bin and UEFI .signed.flash; + // a Smart Component (.fwpkg) answers 200 and flashes nothing, so + // import extracts the flashable file from it. + plan: func(x fwPlanCtx) (*fwRequest, error) { + if x.Caps.simple == "" { + return nil, errors.New("this iLO offers no SimpleUpdate") + } + return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": x.URL}}, nil + }, + formats: []string{".bin", ".signed.flash", ".flash", ".fwpkg"}, + notes: "iLO firmware flashes at once (iLO restarts); UEFI/BIOS is staged and flashed at the next reboot (-reboot). Components only a runtime agent can flash (some NICs) need SUM/iSUT in the OS. Downgrades follow the iLO downgrade policy.", + sources: []string{ + "https://servermanagementportal.ext.hpe.com/docs/references_and_material/blogposts/firmware_updates/part3/firmware_update_part3", + "https://servermanagementportal.ext.hpe.com/docs/redfishservices/ilos/supplementdocuments/updateservice/", + }, + }, + "xcc": { + // /mfwupdate, after taking the push service (HttpPushUriTargetsBusy). + plan: func(x fwPlanCtx) (*fwRequest, error) { + params := obj{"Targets": []string{}} + if x.Apply != "" { + params["@Redfish.OperationApplyTime"] = x.Apply + } + if x.Caps.multipart != "" { + return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params, Claim: true}, nil + } + if x.Caps.push != "" { + return &fwRequest{Method: "push", URI: x.Caps.push, Claim: true}, nil + } + return nil, errors.New("this XCC offers no push update URI") + }, + claims: true, + formats: []string{".uxz", ".zip", ".bin"}, + notes: "Update XCC before UEFI (Lenovo). One push at a time; the busy flag is released after the update.", + sources: []string{ + "https://pubs.lenovo.com/xcc-restapi/simple_update_firmware_post_multi_httppush", + "https://pubs.lenovo.com/xcc2-restapi/prop_firmware_update_service_get", + "https://pubs.lenovo.com/xcc/updating_firmware_overview", + }, + }, + "supermicro": { + // /UpdateService/upload with the documented targets and preserve options. + plan: func(x fwPlanCtx) (*fwRequest, error) { + uri := orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload") + apply := x.applyOr("Immediate") + var params obj + switch x.Component { + case "bios": + params = obj{"Targets": []string{"/redfish/v1/Systems/1/Bios"}, "@Redfish.OperationApplyTime": apply, + "Oem": obj{"Supermicro": obj{"BIOS": obj{"PreserveME": true, "PreserveNVRAM": true, "PreserveSMBIOS": true, "BackupBIOS": false}}}} + case "bmc": + params = obj{"Targets": []string{"/redfish/v1/Managers/1"}, "@Redfish.OperationApplyTime": apply, + "Oem": obj{"Supermicro": obj{"BMC": obj{"PreserveCfg": true, "PreserveSdr": true, "PreserveSsl": true, "BackupBMC": true}}}} + case "cpld": + id := "" + for _, i := range x.Inventory { + if strings.HasPrefix(i, "Motherboard_CPLD_") && id == "" { + id = i + } + } + if id == "" { + return nil, errors.New("the BMC lists no Motherboard_CPLD_* firmware") + } + params = obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + id}, "@Redfish.OperationApplyTime": apply} + default: // NIC, storage AOC, PSU: the BMC picks the device (SFT-DCMS-SINGLE licence) + params = obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply} + } + return &fwRequest{Method: "multipart", URI: uri, Params: params}, nil + }, + notes: "A BIOS update powers the host off first. Gen 12/13 boards take other BIOS preserve options (PreserveOA, PreserveSETUPCONF, ...): set fw_params on the BMC if the BMC refuses these. Redfish updates need the SFT-OOB-LIC or SFT-DCMS-SINGLE licence; NIC/storage/PSU need DCMS. A CPLD update resets the BMC.", + sources: []string{ + "https://www.supermicro.com/manuals/other/RedfishUserGuide.pdf", + "https://github.com/bmc-toolbox/bmclib/blob/main/providers/supermicro/x12.go", + }, + }, + "gigabyte": {plan: amiPlan, formats: []string{".hpm", ".rcu", ".ima_enc", ".ima", ".rbu", ".bin"}, notes: amiNotes, sources: []string{"https://download.gigabyte.com/FileList/Manual/server_manual_redfish_v1.11.0.pdf", "GIGABYTE Firmware Upgrade Guide v0.04 (2022)"}}, + "ami": {plan: amiPlan, notes: amiNotes, sources: []string{"https://download2.mitaccomputing.com/pub/doc/MiTAC_AST2600_Redfish_API_Doc_v.1_051024.pdf", "https://support.blackcoretech.com/support/solutions/articles/70000680445-update-bios-bmc-via-redfish-multipart-push"}}, + "asrockrack": {plan: amiPlan, notes: amiNotes + " ASRock Rack's own Redfish firmware notes could not be read: this is the AMI MegaRAC request, check it on one machine first.", sources: []string{"https://download.asrock.com/TSD/SMU/Manual/ASRock%20Rack%20Server%20Management_v2.8.0.pdf"}}, + "openbmc": { + // update-multipart; the BMC image applies at once (it reboots), the rest OnReset. + plan: func(x fwPlanCtx) (*fwRequest, error) { + params := obj{"Targets": []string{}} + if x.Component == "bmc" { + params = obj{"Targets": []string{"/redfish/v1/Managers/bmc"}, "@Redfish.OperationApplyTime": x.applyOr("Immediate")} + } else if x.Apply != "" { + params["@Redfish.OperationApplyTime"] = x.Apply + } + if x.Caps.multipart != "" { + return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params}, nil + } + if x.Caps.push != "" && x.Component == "bmc" { + return &fwRequest{Method: "push", URI: x.Caps.push}, nil + } + return nil, errors.New("this OpenBMC offers no multipart update (SimpleUpdate is off in bmcweb by default)") + }, + formats: []string{".tar"}, + notes: "Signed image tarballs whose MANIFEST MachineName matches the BMC.", + sources: []string{"https://github.com/openbmc/bmcweb/blob/master/redfish-core/lib/update_service.hpp", "https://github.com/openbmc/docs/blob/master/REDFISH-cheatsheet.md"}, + }, + "xfusion": { + // upload to the BMC's /tmp/web, then SimpleUpdate naming the file + plan: func(x fwPlanCtx) (*fwRequest, error) { + if x.Component != "bmc" && x.Component != "bios" && x.Component != "cpld" { + return nil, fmt.Errorf("iBMC updates BMC, BIOS and CPLD out of band; %s firmware goes through Smart Provisioning in-band (not supported)", x.Component) + } + if x.Caps.simple == "" { + return nil, errors.New("this iBMC offers no SimpleUpdate") + } + return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "file", + Then: &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": "/tmp/web/" + path.Base(x.File)}}}, nil + }, + formats: []string{".hpm"}, + notes: "The BMC restarts after a BMC update.", + sources: []string{ + "https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_manage_file.py", + "https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_outband_fw_update.py", + }, + }, + "kaytus": { + // upload (with the BMC password), then an OEM SimpleUpdate; the tasks have fixed names + plan: func(x fwPlanCtx) (*fwRequest, error) { + oem := obj{"PreserveConf": true} + var tasks []string + switch x.Component { + case "bmc": + oem["FlashItem"], tasks = "OpenBMC_ENC", []string{"/redfish/v1/TaskService/Tasks/BMC"} + case "bios": + oem["FlashItem"], oem["BiosFlash"] = "BIOS", "Both" + tasks = []string{"/redfish/v1/TaskService/Tasks/BIOS0", "/redfish/v1/TaskService/Tasks/BIOS1"} + default: + return nil, fmt.Errorf("KAYTUS: BMC and BIOS only (not %s)", x.Component) + } + return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "fwimage", + Form: map[string]string{"CurrentPassword": x.Password, "EncryptFlag": "False"}, FailText: "Verification Failed", + Then: &fwRequest{Method: "simple", URI: orDefault(x.Caps.simple, "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"), Params: obj{"Oem": obj{"Public": oem}}}, + Tasks: tasks}, nil + }, + formats: []string{".hpm", ".bin"}, + notes: "No public Redfish guide: this is what KAYTUS's own SDK (ksManage) does on X3-generation BMCs; older generations use a non-Redfish API. A BIOS update with the host on waits for a power cycle (-reboot); the BMC restarts after a BMC update.", + sources: []string{"https://github.com/KSManageOSS/ksManage/blob/HEAD/ksmanage_sdk/interface/CommonX3.py", "https://github.com/ieisystem/kaytus.ksmanage"}, + }, + "irmc": { + // OEM actions per component, file in the "data" part + plan: func(x fwPlanCtx) (*fwRequest, error) { + p := "Fsas" // FTS on S5/S6 (Vendor ts_fujitsu or none), Fsas on newer firmware + if x.Vendor == "" || strings.EqualFold(x.Vendor, "ts_fujitsu") { + p = "FTS" + } + switch x.Component { + case "bmc": + return &fwRequest{Method: "multipart", URI: "/redfish/v1/Managers/iRMC/Actions/" + p + "Manager.FWUpdate", + AltURI: "/redfish/v1/Managers/iRMC/Actions/Oem/" + p + "Manager.FWUpdate", NoParams: true, Field: "data"}, nil + case "bios": + return &fwRequest{Method: "multipart", URI: "/redfish/v1/Systems/0/Bios/Actions/Oem/" + p + "Bios.BiosUpdate", NoParams: true, Field: "data"}, nil + } + return nil, fmt.Errorf("iRMC: iRMC and BIOS only (not %s)", x.Component) + }, + formats: []string{".bin", ".upc"}, + notes: "iRMC .bin, BIOS .UPC. With the host on, a BIOS update waits for a reboot (-reboot); the iRMC restarts after its own update.", + sources: []string{"https://github.com/fujitsu/ansible-irmc-integration/blob/HEAD/plugins/modules/irmc_fwbios_update.py", "https://github.com/mmurayama/fujitsu-redfish-samples/blob/master/scripts/update_irmc_firmware.py"}, + }, + "cimc": { + unsupported: "Cisco documents CIMC firmware updates over Redfish only as SimpleUpdate pulls over SCP (and HSU/HUU from NFS, CIFS or HTTPS shares); Rollcall serves packages over HTTP", + sources: []string{"https://www.cisco.com/c/en/us/td/docs/unified_computing/ucs/c/sw/api/4_3/b-cisco-ucs-c-series-servers-rest-api-programmer-s-guide-release-4-3.pdf"}, + }, +} + +const amiNotes = "MegaRAC: the BMC restarts after a BMC flash and accepts no logins while flashing; do not use its web UI meanwhile." + +// amiPlan: /UpdateService/upload with UpdateParameters (the inventory target) +// and OemParameters (ImageType); .hpm packages have their own targets. +func amiPlan(x fwPlanCtx) (*fwRequest, error) { + hpm := strings.HasSuffix(strings.ToLower(x.File), ".hpm") + var target, typ string + switch x.Component { + case "bmc": + if hpm { + target, typ = "BMC", "HPM_BMC" + } else { + target, typ = orDefault(x.first("BMCImage1", "BMC"), "BMC"), "BMC" + } + case "bios": + if hpm { + target, typ = "BIOS2", "HPM_BIOS" + } else { + target, typ = "BIOS", "BIOS" + } + case "cpld": + target, typ = "MB_CPLD1", "MB_CPLD" + default: + return nil, fmt.Errorf("MegaRAC: BMC, BIOS and CPLD (not %s)", x.Component) + } + if len(x.Inventory) > 0 && !x.has(target) { + return nil, fmt.Errorf("the BMC lists no firmware %s (it has %s)", target, strings.Join(x.Inventory, ", ")) + } + params := obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + target}} + if x.Apply != "" { + params["@Redfish.OperationApplyTime"] = x.Apply + } + return &fwRequest{Method: "multipart", URI: orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload"), Params: params, + Parts: []fwPart{{Name: "OemParameters", Body: obj{"ImageType": typ}}}}, nil +} + +// simpleBody is a SimpleUpdate body for the package URL, plus extra fields. +func simpleBody(x fwPlanCtx, extra obj) obj { + b := obj{"ImageURI": x.URL} + if len(x.Caps.protocols) > 0 { + b["TransferProtocol"] = strings.ToUpper(strings.SplitN(x.URL, "://", 2)[0]) + } + for k, v := range extra { + b[k] = v + } + return b +} + +func init() { + for i := range redfishProfiles { + p := &redfishProfiles[i] + if v, ok := fwVendors[p.Driver]; ok { + p.FWPlan, p.FWMethods, p.FWClaims, p.FWUnsupported = v.plan, v.methods, v.claims, v.unsupported + p.FWFormats, p.FWNotes, p.FWSources = v.formats, v.notes, v.sources + } + } +} diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 67af819..9736ef8 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -201,6 +201,12 @@ func (a *App) routes() http.Handler { mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside // disk-based deployment mux.HandleFunc("GET /api/v1/os", a.adminAuth(a.handleOS)) + mux.HandleFunc("GET /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo)) + mux.HandleFunc("POST /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo)) + mux.HandleFunc("DELETE /api/v1/firmware/{pkg}", a.adminAuth(a.handleFirmwareRemove)) + mux.HandleFunc("GET /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware)) + mux.HandleFunc("POST /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware)) + mux.HandleFunc("POST /api/v1/machines/{id}/firmware/update", a.adminAuth(a.handleFirmwareUpdate)) mux.HandleFunc("POST /api/v1/machines/{id}/deploy", a.adminAuth(a.handleDeploy)) mux.HandleFunc("POST /api/v1/machines/{id}/release", a.adminAuth(a.handleRelease)) mux.HandleFunc("GET /api/v1/machines/{id}/jobs", a.adminAuth(a.handleJobs)) @@ -282,6 +288,7 @@ func (a *App) routes() http.Handler { mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir)))) mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir)))) mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir)))) + mux.HandleFunc("GET /firmware/{pkg}/{file}", a.handleFirmwareFile) // BMCs download packages (no listing) // embedded web UI sub, _ := fs.Sub(webFS, "web") mux.Handle("GET /", a.uiOverHTTPS(uiHeaders(http.FileServer(http.FS(sub))))) @@ -795,6 +802,7 @@ func serve(cfgPath string) error { return fmt.Errorf("dns: %w", err) } go app.jobWatchdog() + go app.resumeFirmware() handler := app.routes() if cfg.TLSListen != "" { names := append([]string{}, cfg.TLSNames...) diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index 4de16b1..f30d9d7 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -239,6 +239,9 @@ func openStore(ctx context.Context, url string) (*Store, error) { if _, err := pool.Exec(ctx, dnsRecordsSchema); err != nil { return nil, fmt.Errorf("schema: %w", err) } + if _, err := pool.Exec(ctx, firmwareSchema); err != nil { + return nil, fmt.Errorf("schema: %w", err) + } return &Store{pool: pool}, nil } diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index 12e2b54..857fc59 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -125,7 +125,7 @@ function render() { case 'network': main.innerHTML = pageNetwork(); wireNetwork(); break; case 'tags': main.innerHTML = pageTags(); wireTags(); break; case 'profiles': main.innerHTML = pageProfiles(); wireProfiles(); break; - case 'images': main.innerHTML = pageImages(); break; + case 'images': main.innerHTML = pageImages(); wireImages(); break; default: main.innerHTML = pageOverview(); wireOverview(); } } @@ -453,7 +453,7 @@ function jobsTable(jobs, withMachine) { function pageMachine(d, tab) { const s = d.summary, inv = d.inventory || {}, sys = inv.system || {}, cpu = inv.cpu || {}, mem = inv.memory || {}, st = inv.storage || {}, b = d.bmc || {}; const m = S.machines.find(x => x.id === s.id) || s; - const tabs = ['overview', 'hardware', 'network', 'bmc', 'jobs', 'events', 'raw']; + const tabs = ['overview', 'hardware', 'network', 'bmc', 'firmware', 'jobs', 'events', 'raw']; let body = ''; if (tab === 'overview') { const job = s.job || (d.jobs || [])[0]; const p = (job && job.progress) || {}; const active = job && (job.state === 'pending' || job.state === 'running'); @@ -501,7 +501,8 @@ function pageMachine(d, tab) { ['Source', esc(b.source)], ['Address', `${esc(b.address)}${b.port && b.port !== 623 ? ':' + esc(b.port) : ''}`], ['MAC', `${esc(b.mac)}`], ['User', esc(b.user)], ['Password', d.has_bmc_secret ? 'stored encrypted on the server' : 'none'], ['Configured', b.configured ? 'yes' : 'no'], ['Verified out-of-band', b.verified_oob === true ? 'yes' : b.verified_oob === false ? 'no' : ''], ['Channel', esc(b.channel)], ['LAN', b.lan ? `${esc(JSON.stringify(b.lan))}` : ''], ['Firmware', esc((b.mc_info || {})['Firmware Revision'])], ['Manufacturer', esc((b.mc_info || {})['Manufacturer Name'])], ['Errors', (b.errors || []).length ? `${esc(b.errors.join('; '))}` : '']]))} ${(b.users_before || []).length ? card('User table before commissioning', table(['ID', 'Name', 'Priv', 'IPMI'], b.users_before.map(u => [esc(u.id), esc(u.name), esc(u.priv), u.ipmi_msg ? 'yes' : 'no']))) : ''} ${b.fru ? card('FRU', `
${esc(JSON.stringify(b.fru, null, 1))}
`) : ''}`; - } else if (tab === 'jobs') body = jobsTable(d.jobs || [], false); + } else if (tab === 'firmware') body = `
Packages come from the repository (OS images page); the update runs on rack controller ${esc(s.rack_controller || '')}
loading…
`; + else if (tab === 'jobs') body = jobsTable(d.jobs || [], false); else if (tab === 'events') body = table(['Time', 'Event', 'Detail'], (d.events || []).slice().reverse().map(e => [esc(when(e.ts)), esc(e.event), `${esc(e.detail)}`])); else body = `
${esc(JSON.stringify(d, null, 1))}
`; @@ -522,6 +523,7 @@ function pageMachine(d, tab) { function wireMachine(d) { const id = d.summary.id, ids = [id]; if (route().tab === 'network') wireNetPlan(id); + if (route().tab === 'firmware') loadFirmware(id, false); wireMenus(); $$('#dpower [data-item]').forEach(b => b.onclick = () => act.power(ids, b.dataset.item)); $$('#dboot [data-item]').forEach(b => b.onclick = () => act.bootdev(ids, b.dataset.item)); @@ -535,6 +537,54 @@ function wireMachine(d) { $$('[data-untag]').forEach(a => a.onclick = ev => { ev.preventDefault(); each(ids, 'untag', async () => (await api('POST', `/api/v1/machines/${id}/tags`, { remove: [a.dataset.untag] })).effective.join(' ')); }); } +// ---------------------------------------------------------------- firmware +async function loadFirmware(id, read) { + const box = $('#fwbox'); if (!box) return; + const btn = $('#fw-read'); if (btn) btn.onclick = () => loadFirmware(id, true); + if (read) box.innerHTML = 'reading the inventory from the BMC…'; + let d; + try { d = await api(read ? 'POST' : 'GET', `/api/v1/machines/${id}/firmware`); } catch (e) { box.innerHTML = `${esc(e.message)}`; return; } + const inv = (d.inventory || {}).components || []; + const offers = d.offers || []; + box.className = ''; + box.innerHTML = `
+ ${card(`Installed · ${inv.length} components${d.inventory && d.inventory.at && inv.length ? ` · read ${esc(ago(d.inventory.at))} ago` : ''}`, inv.length ? table(['Component', 'Name', 'Version', 'Updateable'], inv.map(c => [esc(c.component), esc(c.name), `${esc(c.version)}`, c.updateable ? 'yes' : 'no'])) : 'not read yet: Read from BMC', true)} + ${card(`Repository for ${esc(d.driver || '?')} ${esc(d.model || '')}`, offers.length ? table(['Component', 'Version', 'Installed now', '', ''], offers.map(o => [esc(o.package.component), `${esc(o.package.version)}`, `${esc(o.installed || '—')}`, o.same ? 'installed' : 'differs', ``])) : 'no package in the repository for this vendor and model', true)} +
`; + $$('[data-fwpkg]').forEach(b => b.onclick = () => fwUpdate(id, offers.find(o => o.package.id === b.dataset.fwpkg).package)); +} +async function fwUpdate(id, p) { + const v = await dialog(`Update ${p.component} to ${p.version}`, [ + { name: 'apply', label: 'Apply', type: 'select', value: '', options: [['', "the BMC's default"], ['OnReset', 'at the next reset'], ['Immediate', 'immediately']] }, + { name: 'reboot', label: 'Power-cycle the host afterwards and verify the version', type: 'checkbox', value: p.component === 'bios' }, + { name: 'force', label: 'Send even if vendor or model does not match', type: 'checkbox', value: false }, + ], `${p.file} (${human(p.size)}). The BMC checks the package itself; an update cannot be cancelled once the BMC has it. A BMC update restarts the BMC for a few minutes.`, 'Start update'); + if (!v) return; + try { const r = await api('POST', `/api/v1/machines/${id}/firmware/update`, { package: p.id, apply: v.apply, reboot: v.reboot, force: v.force }); toast(`firmware job ${r.job} started`); await loadDetail(id); } catch (e) { toast(e.message, true); } +} +function wireImages() { + const el = $('#fwrepo'); if (!el) return; + const show = () => api('GET', '/api/v1/firmware').then(ps => { + el.className = ''; + el.innerHTML = ps.length ? table(['Package', 'Vendor', 'Component', 'Version', 'Models', 'Size', 'Imported', ''], ps.map(p => [`${esc(p.id)}
${esc(p.file)}`, esc(p.vendor), esc(p.component), `${esc(p.version)}`, esc((p.models || []).join(', ') || 'any'), human(p.size), esc(when(p.imported)), ``])) : 'no packages yet'; + $$('[data-fwrm]').forEach(b => b.onclick = async () => { if (confirm(`Remove ${b.dataset.fwrm}?`)) { try { await api('DELETE', '/api/v1/firmware/' + b.dataset.fwrm); show(); } catch (e) { toast(e.message, true); } } }); + }).catch(e => { el.innerHTML = `${esc(e.message)}`; }); + show(); + $('#fwimport').onclick = async () => { + const drivers = await api('GET', '/api/v1/power-drivers').then(ds => ds.filter(d => d.family === 'redfish' && d.name !== 'redfish').map(d => [d.name, d.description]), () => []); + const v = await dialog('Import a firmware package', [ + { name: 'url', label: 'Download URL', placeholder: 'https://dl.dell.com/FOLDER.../BIOS_XXXX_WN64_2.19.1.EXE' }, + { name: 'vendor', label: 'Vendor (BMC driver)', type: 'select', options: drivers }, + { name: 'component', label: 'Component', type: 'select', options: ['bios', 'bmc', 'nic', 'raid', 'cpld', 'psu', 'drive', 'other'].map(c => [c, c]) }, + { name: 'version', label: 'Version', placeholder: 'as the component reports it once installed' }, + { name: 'models', label: 'Models', placeholder: 'R650, R750 (empty: any model of this vendor)' }, + { name: 'sha256', label: 'SHA-256', placeholder: "from the vendor's download page (recommended)" }, + ], 'The region downloads the package; rack controllers serve it to the BMCs. A local file: rollcall firmware import FILE on the region.', 'Import'); + if (!v) return; + try { const p = await api('POST', '/api/v1/firmware', { url: v.url.trim(), vendor: v.vendor, component: v.component, version: v.version.trim(), models: v.models.split(',').map(x => x.trim()).filter(Boolean), sha256: v.sha256.trim() }); toast(`imported ${p.id}`); show(); } catch (e) { toast(e.message, true); } + }; +} + // ---------------------------------------------------------------- jobs function pageJobs() { const active = S.jobs.filter(j => j.state === 'pending' || j.state === 'running'), rest = S.jobs.filter(j => !(j.state === 'pending' || j.state === 'running')); @@ -638,6 +688,9 @@ function wireProfiles() { api('GET', '/api/v1/images').then(imgs => { const el = $('#rootimgs'); if (el) el.innerHTML = table(['Image', 'Complete', 'Squashfs', 'Built'], imgs.map(i => [esc(i.name), i.complete ? 'yes' : 'no', human(i['filesystem.squashfs']), esc(when(i.built))])); }).catch(() => { }); } function pageImages() { + return pageOSImages() + `

Firmware packages

loading…

Updates run from a machine's Firmware tab, or rollcall firmware update MACHINE|tag:T PACKAGE.

`; +} +function pageOSImages() { return `

OS images

disk images for deployment · refreshed nightly by rollcall os sync from the vendors' checksum files
${table(['OS', 'Current version', 'Family', 'Compressed', 'Disk needed', 'Imported', 'Track', 'Keep', 'Versions', 'Status'], S.os.map(i => { const c = i.current || {}; const s = i.source || {}; return [`${esc(i.name)}${s.description ? `
${esc(s.description)}` : ''}`, `${esc(c.version || '—')}`, esc(s.family || c.family), c.size ? human(c.size) : '', c.raw_size ? human(c.raw_size) : '', c.imported ? `${esc(when(c.imported))} (${ago(c.imported)})` : '', esc(s.track || ''), s.keep || '', (i.versions || []).map(v => `${esc(v.version)}`).join('
'), i.warning ? `${esc(i.warning)}` : 'ok']; }))}

Pinned sources keep current until rollcall os promote NAME VERSION; a deployment always uses the exact version it started with. Import a local file with rollcall os import NAME FILE.

`; diff --git a/lab/pve-vbmc/pve_redfish.py b/lab/pve-vbmc/pve_redfish.py index 5c87b05..8fc3c06 100644 --- a/lab/pve-vbmc/pve_redfish.py +++ b/lab/pve-vbmc/pve_redfish.py @@ -11,9 +11,19 @@ idrac / ilo / generic drivers see what real hardware shows: Config: the pve-vbmc config.json; a bmcs entry may carry "redfish": "idrac"|"ilo"|"generic". Basic auth with the same user/password as the IPMI side. Self-signed TLS. + +Firmware (UpdateService), for testing rollcall's firmware jobs: idrac takes +multipart uploads and SimpleUpdate, ilo only SimpleUpdate (it downloads the +ImageURI itself), generic multipart. A "package" is any file with a line +"version=X"; a name with bios/rom/flash in it is the BIOS, anything else the +BMC. The BMC version changes when its task completes; the BIOS one at the next +reset of the VM, as on real servers. Versions are kept in +/var/lib/pve-vbmc/fw-.json. """ import argparse import base64 +import email +import email.policy import ipaddress import json import logging @@ -35,11 +45,16 @@ log = logging.getLogger("pve-redfish") PROFILES = { "idrac": {"vendor": "Dell", "manufacturer": "Dell Inc.", "model": "PowerEdge R760 (pve)", "id": "System.Embedded.1", "resets": ["On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"], - "actioninfo": False, "etag": True}, + "actioninfo": False, "etag": True, "fw": {"multipart": "/redfish/v1/UpdateService/MultipartUpload", "simple": True, + "ids": {"bmc": "Installed-25227", "bios": "Installed-159"}, + "names": {"bmc": "Integrated Dell Remote Access Controller", "bios": "BIOS"}}}, "ilo": {"vendor": "HPE", "manufacturer": "HPE", "model": "ProLiant DL380 Gen11 (pve)", "id": "1", - "resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False}, + "resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False, + "fw": {"multipart": "", "simple": True, "ids": {"bmc": "1", "bios": "2"}, "names": {"bmc": "iLO 6", "bios": "System ROM"}}}, "generic": {"vendor": "", "manufacturer": "QEMU", "model": "Standard PC (pve)", "id": "1", - "resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False}, + "resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False, + "fw": {"multipart": "/redfish/v1/UpdateService/update-multipart", "simple": False, "ids": {"bmc": "bmc", "bios": "bios"}, + "names": {"bmc": "BMC firmware", "bios": "BIOS"}}}, } @@ -82,7 +97,67 @@ class Pve: return order -def make_handler(pve, prof, user, password): +class Firmware: + """Installed versions, staged BIOS and update tasks of one emulated BMC.""" + + def __init__(self, vmid, prof): + self.path = f"/var/lib/pve-vbmc/fw-{vmid}.json" + self.prof = prof["fw"] + self.lock = threading.Lock() + self.tasks = {} + try: + self.v = json.load(open(self.path)) + except (OSError, ValueError): + self.v = {"bmc": "1.0.0", "bios": "1.0.0", "staged_bios": ""} + + def save(self): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + json.dump(self.v, open(self.path, "w")) + + def inventory(self): + return [{"@odata.id": f"/redfish/v1/UpdateService/FirmwareInventory/{self.prof['ids'][c]}-{self.v[c]}", "Id": f"{self.prof['ids'][c]}-{self.v[c]}", + "Name": self.prof["names"][c], "Version": self.v[c], "Updateable": True} for c in ("bmc", "bios")] + + def start(self, name, data): + """Begin flashing; returns the task id.""" + comp = "bios" if any(w in name.lower() for w in ("bios", "rom", "flash")) else "bmc" + ver = "" + for line in data.decode(errors="replace").splitlines(): + if line.startswith("version="): + ver = line.split("=", 1)[1].strip() + with self.lock: + tid = str(len(self.tasks) + 1) + self.tasks[tid] = {"state": "Running", "pct": 0, "msg": f"flashing {comp} {ver}"} + threading.Thread(target=self.flash, args=(tid, comp, ver), daemon=True).start() + log.info("firmware task %s: %s -> %s (%s)", tid, comp, ver or "?", name) + return tid + + def flash(self, tid, comp, ver): + for pct in (10, 30, 60, 90): + time.sleep(4) + self.tasks[tid]["pct"] = pct + t = self.tasks[tid] + if not ver: + t.update(state="Exception", msg="the image is not valid for this system") + return + with self.lock: + if comp == "bmc": + self.v["bmc"] = ver + t.update(state="Completed", pct=100, msg=f"BMC firmware {ver} installed") + else: + self.v["staged_bios"] = ver + t.update(state="Completed", pct=100, msg=f"BIOS {ver} staged; it applies at the next reboot") + self.save() + + def reset(self): + with self.lock: + if self.v.get("staged_bios"): + log.info("reset: BIOS %s applied", self.v["staged_bios"]) + self.v["bios"], self.v["staged_bios"] = self.v["staged_bios"], "" + self.save() + + +def make_handler(pve, prof, user, password, fw): sysid = prof["id"] syspath = f"/redfish/v1/Systems/{sysid}" state = {"override": "None", "enabled": "Disabled", "etag": 1} @@ -150,6 +225,24 @@ def make_handler(pve, prof, user, password): return self.send(200, self.system(), hdr) if p == syspath + "/ResetActionInfo": return self.send(200, {"Parameters": [{"Name": "ResetType", "Required": True, "AllowableValues": prof["resets"]}]}) + if p == "/redfish/v1/UpdateService": + us = {"@odata.id": p, "ServiceEnabled": True, "FirmwareInventory": {"@odata.id": p + "/FirmwareInventory"}} + if fw.prof["multipart"]: + us["MultipartHttpPushUri"] = fw.prof["multipart"] + if fw.prof["simple"]: + us["Actions"] = {"#UpdateService.SimpleUpdate": {"target": p + "/Actions/UpdateService.SimpleUpdate", + "TransferProtocol@Redfish.AllowableValues": ["HTTP", "HTTPS"]}} + return self.send(200, us) + if p == "/redfish/v1/UpdateService/FirmwareInventory": + return self.send(200, {"Members": [{"@odata.id": e["@odata.id"]} for e in fw.inventory()]}) + if p.startswith("/redfish/v1/UpdateService/FirmwareInventory/"): + for e in fw.inventory(): + if e["@odata.id"] == p: + return self.send(200, e) + if p.startswith("/redfish/v1/TaskService/Tasks/"): + t = fw.tasks.get(p.rsplit("/", 1)[1]) + if t: + return self.send(200, {"@odata.id": p, "TaskState": t["state"], "PercentComplete": t["pct"], "Messages": [{"Message": t["msg"]}]}) self.error(404, "not found") def do_PATCH(self): @@ -170,10 +263,33 @@ def make_handler(pve, prof, user, password): state["etag"] += 1 self.send(204) + def task(self, tid): + loc = f"/redfish/v1/TaskService/Tasks/{tid}" + self.send(202, {"@odata.id": loc, "@odata.type": "#Task.v1_4_0.Task", "TaskState": "Running"}, {"Location": loc}) + def do_POST(self): if not self.authed(): return - if self.path.rstrip("/") != syspath + "/Actions/ComputerSystem.Reset": + p = self.path.split("?")[0].rstrip("/") + if fw.prof["multipart"] and p == fw.prof["multipart"]: + n = int(self.headers.get("Content-Length") or 0) + if n <= 0: + return self.error(411, "Content-Length required") + raw = self.rfile.read(n) + msg = email.message_from_bytes(b"Content-Type: " + self.headers["Content-Type"].encode() + b"\r\n\r\n" + raw, policy=email.policy.default) + for part in msg.iter_parts(): + if part.get_param("name", header="content-disposition") == "UpdateFile": + return self.task(fw.start(part.get_filename() or "image", part.get_payload(decode=True))) + return self.error(400, "no UpdateFile part") + if fw.prof["simple"] and p == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate": + uri = self.body().get("ImageURI", "") + try: + r = requests.get(uri, timeout=60) + r.raise_for_status() + except requests.RequestException as e: + return self.error(400, f"cannot download {uri}: {e}") + return self.task(fw.start(uri.rsplit("/", 1)[-1], r.content)) + if p != syspath + "/Actions/ComputerSystem.Reset": return self.error(404, "not found") t = self.body().get("ResetType", "") if t not in prof["resets"]: @@ -190,6 +306,8 @@ def make_handler(pve, prof, user, password): pve.post("status/shutdown", forceStop=1, timeout=60) elif t in ("ForceRestart", "GracefulRestart", "PowerCycle"): pve.post("status/reset" if on else "status/start") + if t in ("On", "ForceRestart", "GracefulRestart", "PowerCycle", "PushPowerButton"): + fw.reset() # a staged BIOS applies when the host starts again self.send(204) return H @@ -223,7 +341,7 @@ def main(): addr = entry["address"] user = entry.get("user", cfg.get("user", "admin")) password = entry.get("password", cfg.get("password", "password")) - handler = make_handler(Pve(cfg, args.vmid), prof, user, password) + handler = make_handler(Pve(cfg, args.vmid), prof, user, password, Firmware(args.vmid, prof)) ip = ipaddress.ip_address(addr) cls = Server6 if ip.version == 6 else ThreadingHTTPServer srv = cls((addr, int(entry.get("redfish_port", 443))), handler)