package main // Firmware over Redfish (UpdateService), run where the BMC is reachable (the // rack controller). Three operations, each short enough for one rack command: // // fw-inventory what is installed (FirmwareInventory) // fw-start hand a package to the BMC; returns the task to watch // fw-poll one look at that task // // The package reaches the BMC one of three ways, chosen from what the // UpdateService offers and the vendor profile's order: // // multipart POST MultipartHttpPushUri: UpdateParameters (JSON) + UpdateFile // simple UpdateService.SimpleUpdate with an ImageURI the BMC downloads // from the rack controller (or firmware_base) // push POST HttpPushUri with the file as the body // // The rack controller fetches the package from its own cache (filled from the // region) and sends it with a known length: many BMC web servers refuse // chunked uploads. import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "mime/multipart" "net/http" "net/textproto" "os" "path" "regexp" "sort" "strconv" "strings" "time" ) // fwComponent is one entry of a machine's firmware inventory. type fwComponent struct { ID string `json:"id"` Name string `json:"name"` Version string `json:"version"` Component string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other Updateable bool `json:"updateable"` SoftwareID string `json:"software_id,omitempty"` } // fwStartArg is what fw-start receives. type fwStartArg struct { URL string `json:"url"` // where the package is (the BMC pulls it, or the rack downloads it to push) Component string `json:"component"` // bios | bmc | ... (vendor plans pick targets by it) File string `json:"file"` // its file name Method string `json:"method,omitempty"` // multipart | simple | push; "" = profile order Apply string `json:"apply,omitempty"` // Immediate | OnReset; "" = the BMC's default Targets []string `json:"targets,omitempty"` // inventory resources to update; usually empty } type fwStartResult struct { Method string `json:"method"` Task string `json:"task,omitempty"` // task monitor or task resource; "" when the BMC answered synchronously Note string `json:"note,omitempty"` } // fwTimeouts bound each firmware operation; an upload to a slow BMC takes long. var fwTimeouts = map[string]time.Duration{"fw-inventory": 5 * time.Minute, "fw-start": 45 * time.Minute, "fw-poll": 2 * time.Minute, "fw-finish": time.Minute} type fwPollResult struct { State string `json:"state"` // running | done | failed | unknown Pct int `json:"pct"` Msg string `json:"msg,omitempty"` } // fwKinds sorts inventory entries by id and name, most specific first: a RAID // "controller" is not the management controller. var fwKinds = []struct { kind string re *regexp.Regexp }{ {"bios", regexp.MustCompile(`(?i)\b(bios|uefi|system rom)\b`)}, {"raid", regexp.MustCompile(`(?i)\b(raid|perc|smart ?array|megaraid|hba|storage controller|boss)\b`)}, {"nic", regexp.MustCompile(`(?i)\b(nic|ethernet|network|connectx|mellanox|infiniband|e810|x710|bcm\d*|broadcom adv)`)}, {"psu", regexp.MustCompile(`(?i)\b(psu|power supply)\b`)}, {"cpld", regexp.MustCompile(`(?i)\b(cpld|fpga)\b`)}, {"drive", regexp.MustCompile(`(?i)\b(drive|disk|ssd|nvme|hdd)\b`)}, {"bmc", regexp.MustCompile(`(?i)\b(bmc|idrac|ilo ?\d?|xcc2?|imm2?|ibmc|irmc|cimc|remote access controller|management controller|baseboard management)\b`)}, } // fwKind sorts an inventory entry into a component class by its id and name. func fwKind(id, name string) string { s := id + " " + name for _, k := range fwKinds { if k.re.MatchString(s) { return k.kind } } return "other" } func firmwareRun(ctx context.Context, c *bmcCreds, op, arg string) (string, error) { d, _ := driverByName(c.Driver) if d.Family != "redfish" { return "", fmt.Errorf("firmware updates need a Redfish driver (this BMC uses %s)", orDefault(c.Driver, "ipmi")) } prof := redfishProfileFor(c.Driver) r := newRF(c) r.hc.Timeout = 0 // uploads take minutes; the context bounds each call if err := r.open(ctx, c, prof); err != nil { return "", err } defer r.logout() var out any var err error switch op { case "fw-inventory": out, err = r.fwInventory(ctx) case "fw-start": var a fwStartArg if err := json.Unmarshal([]byte(arg), &a); err != nil { return "", fmt.Errorf("fw-start: %w", err) } out, err = r.fwStart(ctx, c, prof, a) case "fw-poll": out, err = r.fwPoll(ctx, arg) case "fw-finish": // after the task ended: give back what fw-start claimed if prof.FWClaims { r.fwRelease(ctx) } out = obj{"ok": true} default: return "", fmt.Errorf("unknown firmware operation %q", op) } if err != nil { return "", err } b, _ := json.Marshal(out) return string(b), nil } func (r *rfClient) fwInventory(ctx context.Context) ([]fwComponent, error) { var coll struct { Members []struct { ID string `json:"@odata.id"` } `json:"Members"` } if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err != nil { return nil, err } out := []fwComponent{} for i, mem := range coll.Members { if i >= 300 { break } id := path.Base(strings.TrimRight(mem.ID, "/")) // Dell lists what is installed and what was before or is staged if strings.HasPrefix(id, "Previous-") || strings.HasPrefix(id, "Available-") { continue } var e obj if _, err := r.do(ctx, "GET", mem.ID, nil, &e); err != nil { continue } up, _ := e["Updateable"].(bool) fc := fwComponent{ID: str(e["Id"]), Name: str(e["Name"]), Version: str(e["Version"]), Updateable: up, SoftwareID: str(e["SoftwareId"])} if fc.ID == "" { fc.ID = id } fc.Component = fwKind(fc.ID, fc.Name) out = append(out, fc) } sort.Slice(out, func(i, j int) bool { if out[i].Component != out[j].Component { return out[i].Component < out[j].Component } return out[i].ID < out[j].ID }) return out, nil } // fwCaps is what an UpdateService offers. type fwCaps struct { simple, multipart, push string protocols []string } func (r *rfClient) fwCaps(ctx context.Context) (fwCaps, error) { var us obj if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil { return fwCaps{}, err } act := asObj(dig(us, "Actions", "#UpdateService.SimpleUpdate")) if r.pushURIs == nil { r.pushURIs = map[string]bool{} } for _, k := range []string{"MultipartHttpPushUri", "HttpPushUri"} { if u := str(us[k]); u != "" && !strings.Contains(u, "://") { r.pushURIs[u] = true } } return fwCaps{ simple: str(act["target"]), protocols: strList(act["TransferProtocol@Redfish.AllowableValues"]), multipart: str(us["MultipartHttpPushUri"]), push: str(us["HttpPushUri"]), }, nil } // fwRequest is one way to hand a package over, as a vendor documents it. type fwRequest struct { Method string // multipart | simple | push: how this request carries the package URI string // where it goes AltURI string // tried when URI answers 404 (paths that moved between firmware generations) Params obj // multipart UpdateParameters, or the JSON body of a simple request NoParams bool // multipart without an UpdateParameters part Parts []fwPart // more parts before the file (AMI OemParameters) Field string // the file's form field (default UpdateFile) Form map[string]string // text fields after the file FailText string // a 2xx answer that contains this is a refusal (KAYTUS "Verification Failed") Claim bool // take HttpPushUriTargetsBusy first (Lenovo); fw-finish gives it back Then *fwRequest // a second call naming the uploaded file (xFusion, KAYTUS) Tasks []string // fixed tasks to watch when the answers name none (KAYTUS) } type fwPart struct { Name string Body obj } // fwPlanCtx is what a vendor plan decides from. type fwPlanCtx struct { Component, File, URL, Apply, Password string Vendor string // ServiceRoot Vendor Caps fwCaps // what the UpdateService offers Inventory []string // FirmwareInventory member ids } func (x fwPlanCtx) has(id string) bool { return hasTag(x.Inventory, id) } func (x fwPlanCtx) first(ids ...string) string { for _, id := range ids { if x.has(id) { return id } } return "" } func (x fwPlanCtx) applyOr(def string) string { return orDefault(x.Apply, def) } func (r *rfClient) fwStart(ctx context.Context, c *bmcCreds, prof redfishProfile, a fwStartArg) (*fwStartResult, error) { caps, err := r.fwCaps(ctx) if err != nil { return nil, fmt.Errorf("no UpdateService: %w", err) } forced := orDefault(a.Method, c.Extra["fw_method"]) if prof.FWUnsupported != "" && forced == "" { return nil, errors.New(prof.FWUnsupported + " (to try a standard method anyway: -method multipart|simple|push)") } if prof.FWPlan != nil && forced == "" { x := fwPlanCtx{Component: a.Component, File: a.File, URL: a.URL, Apply: a.Apply, Password: r.pass, Caps: caps} var root obj if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err == nil { x.Vendor = str(root["Vendor"]) } var coll struct { Members []struct { ID string `json:"@odata.id"` } `json:"Members"` } if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err == nil { for _, m := range coll.Members { x.Inventory = append(x.Inventory, path.Base(strings.TrimRight(m.ID, "/"))) } } req, err := prof.FWPlan(x) if err != nil { return nil, err } if raw := c.Extra["fw_params"]; raw != "" { // the operator's own UpdateParameters / body var o obj if err := json.Unmarshal([]byte(raw), &o); err != nil { return nil, fmt.Errorf("fw_params is not JSON: %w", err) } req.Params = o } return r.fwExec(ctx, req, a) } order := prof.FWMethods if len(order) == 0 { order = []string{"multipart", "simple", "push"} } if forced != "" { order = []string{forced} } var tried []string for _, m := range order { req := fwStandard(caps, m, a) if req == nil { continue } res, err := r.fwExec(ctx, req, a) if err == nil { return res, nil } // a BMC that does not like this way gets the next one; a refused // package (wrong model, signature, downgrade) stops here if e := asRFError(err); e == nil || (e.code != http.StatusMethodNotAllowed && e.code != http.StatusNotImplemented && e.code != http.StatusUnsupportedMediaType) { return nil, fmt.Errorf("%s: %w", m, err) } tried = append(tried, m+": "+err.Error()) } if len(tried) > 0 { return nil, fmt.Errorf("the BMC took no upload method: %s", strings.Join(tried, "; ")) } return nil, fmt.Errorf("the BMC's UpdateService offers none of %v", order) } // fwStandard is the DMTF request for a method, nil when the service lacks it. func fwStandard(caps fwCaps, method string, a fwStartArg) *fwRequest { switch method { case "multipart": if caps.multipart == "" { return nil } params := obj{"Targets": orEmpty(a.Targets)} if a.Apply != "" { params["@Redfish.OperationApplyTime"] = a.Apply } return &fwRequest{Method: "multipart", URI: caps.multipart, Params: params} case "simple": if caps.simple == "" { return nil } body := obj{"ImageURI": a.URL} if len(caps.protocols) > 0 { // only when the service lists them: others want the scheme in the URI body["TransferProtocol"] = strings.ToUpper(strings.SplitN(a.URL, "://", 2)[0]) } if len(a.Targets) > 0 { body["Targets"] = a.Targets } return &fwRequest{Method: "simple", URI: caps.simple, Params: body} case "push": if caps.push == "" { return nil } return &fwRequest{Method: "push", URI: caps.push} } return nil } // fwExec carries out a request (and its follow-up) and names the task to watch. func (r *rfClient) fwExec(ctx context.Context, req *fwRequest, a fwStartArg) (*fwStartResult, error) { if req.Claim { var us obj if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil { return nil, err } if us["HttpPushUriTargetsBusy"] == true { return nil, errors.New("the BMC's update service is busy with another upload (HttpPushUriTargetsBusy)") } if _, err := r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": true}); err != nil { return nil, fmt.Errorf("claim the update service: %w", err) } } resp, err := r.fwSend(ctx, req, a) if err != nil { if req.Claim { r.fwRelease(ctx) } return nil, err } res := &fwStartResult{Method: req.Method, Task: fwTaskOf(resp)} if req.Then != nil { tresp, err := r.fwSend(ctx, req.Then, a) if err != nil { return nil, fmt.Errorf("the BMC has the file, but did not start the update: %w", err) } res.Task = fwTaskOf(tresp) } if res.Task == "" && len(req.Tasks) > 0 { res.Task = strings.Join(req.Tasks, ",") } return res, nil } func (r *rfClient) fwRelease(ctx context.Context) { _, _ = r.call(ctx, "GET", "/redfish/v1/UpdateService", nil, "") _, _ = r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": false}) } func (r *rfClient) fwSend(ctx context.Context, req *fwRequest, a fwStartArg) (*rfResp, error) { var resp *rfResp var err error send := func(uri string) (*rfResp, error) { switch req.Method { case "simple": if proto := str(req.Params["TransferProtocol"]); proto != "" && len(a.URL) > 0 { // checked here so a service that pulls only over NFS/SCP says so clearly if caps, cerr := r.fwCaps(ctx); cerr == nil && len(caps.protocols) > 0 && !hasTag(caps.protocols, proto) { return nil, &rfError{method: "POST", path: uri, code: http.StatusNotImplemented, msg: fmt.Sprintf("transfer protocols %v, not %s", caps.protocols, proto)} } } return r.call(ctx, "POST", uri, req.Params, "") case "multipart", "push": f, size, ferr := fwFetch(ctx, a.URL) if ferr != nil { return nil, ferr } defer os.Remove(f.Name()) defer f.Close() if req.Method == "push" { return r.upload(ctx, uri, "application/octet-stream", f, size, r.etags[uri]) } return r.multipartUpload(ctx, uri, req, a.File, f, size) } return nil, fmt.Errorf("unknown method %q", req.Method) } resp, err = send(req.URI) if e := asRFError(err); e != nil && e.code == http.StatusNotFound && req.AltURI != "" { resp, err = send(req.AltURI) } if err == nil && req.FailText != "" && strings.Contains(string(resp.body), req.FailText) { return nil, fmt.Errorf("the BMC refused the package: %s", req.FailText) } return resp, err } func orEmpty(s []string) []string { if s == nil { return []string{} } return s } // fwTaskOf finds the task in an answer: Location, a Task body, or a Dell job id. func fwTaskOf(resp *rfResp) string { if resp == nil { return "" } if l := resp.hdr.Get("Location"); l != "" { return l } var t struct { ID string `json:"@odata.id"` Type string `json:"@odata.type"` } if json.Unmarshal(resp.body, &t) == nil && strings.Contains(t.Type, "Task") { return t.ID } return "" } // fwFetch downloads the package to a temporary file (the rack's cache serves it). func fwFetch(ctx context.Context, url string) (*os.File, int64, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return nil, 0, err } resp, err := http.DefaultClient.Do(req) if err != nil { return nil, 0, fmt.Errorf("package: %w", err) } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, 0, fmt.Errorf("package %s: %s", url, resp.Status) } f, err := os.CreateTemp("", "rollcall-fw-*") if err != nil { return nil, 0, err } n, err := io.Copy(f, resp.Body) if err == nil { _, err = f.Seek(0, io.SeekStart) } if err != nil { f.Close() os.Remove(f.Name()) return nil, 0, err } return f, n, nil } // upload POSTs a body of known length (no chunked encoding). func (r *rfClient) upload(ctx context.Context, target, ctype string, body io.Reader, size int64, ifMatch string) (*rfResp, error) { p, err := r.resolve(target) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, "POST", r.base+p, body) if err != nil { return nil, err } req.ContentLength = size req.Header.Set("Content-Type", ctype) req.Header.Set("Accept", "application/json") if ifMatch != "" { req.Header.Set("If-Match", ifMatch) } if r.token != "" { req.Header.Set("X-Auth-Token", r.token) } else if r.user != "" { req.SetBasicAuth(r.user, r.pass) } resp, err := r.hc.Do(req) if err != nil { return nil, err } defer resp.Body.Close() data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) out := &rfResp{code: resp.StatusCode, hdr: resp.Header, body: data} if resp.StatusCode >= 300 { ids, msg := redfishMessage(data) return out, &rfError{method: "POST", path: p, code: resp.StatusCode, ids: ids, msg: resp.Status + " " + msg} } return out, nil } // multipartUpload sends UpdateParameters, any OEM parts, the file and text // fields (DSP0266 "multipart HTTP push", plus what vendors add to it). func (r *rfClient) multipartUpload(ctx context.Context, target string, req *fwRequest, name string, f io.Reader, size int64) (*rfResp, error) { var head, after bytes.Buffer mw := multipart.NewWriter(&head) jsonPart := func(n string, v any) { h := textproto.MIMEHeader{} h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q`, n)) h.Set("Content-Type", "application/json") w, _ := mw.CreatePart(h) b, _ := json.Marshal(v) _, _ = w.Write(b) } if !req.NoParams { p := req.Params if p == nil { p = obj{"Targets": []string{}} } jsonPart("UpdateParameters", p) } for _, p := range req.Parts { jsonPart(p.Name, p.Body) } fh := textproto.MIMEHeader{} fh.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, orDefault(req.Field, "UpdateFile"), path.Base(name))) fh.Set("Content-Type", "application/octet-stream") _, _ = mw.CreatePart(fh) // text fields and the closing boundary come after the file keys := make([]string, 0, len(req.Form)) for k := range req.Form { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { fmt.Fprintf(&after, "\r\n--%s\r\nContent-Disposition: form-data; name=%q\r\n\r\n%s", mw.Boundary(), k, req.Form[k]) } fmt.Fprintf(&after, "\r\n--%s--\r\n", mw.Boundary()) body := io.MultiReader(&head, f, &after) return r.upload(ctx, target, mw.FormDataContentType(), body, int64(head.Len())+size+int64(after.Len()), r.etags[target]) } // fwPoll looks at the task(s) of an update; several (comma-separated) are // watched in order and count as done when all are. func (r *rfClient) fwPoll(ctx context.Context, tasks string) (*fwPollResult, error) { if tasks == "" { return &fwPollResult{State: "unknown"}, nil } var last *fwPollResult seen := false for _, t := range strings.Split(tasks, ",") { res, err := r.fwPollOne(ctx, t) if err != nil { return nil, err } if res.State == "unknown" { // not (or no longer) listed: the others decide continue } seen = true if res.State != "done" { return res, nil } last = res } if !seen { return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil } return last, nil } var fwScheduledRe = regexp.MustCompile(`(?i)\bscheduled\b|completed successfully`) func (r *rfClient) fwPollOne(ctx context.Context, task string) (*fwPollResult, error) { resp, err := r.call(ctx, "GET", task, nil, "") if e := asRFError(err); e != nil && e.code == http.StatusNotFound { // iDRAC: while it updates itself the task moves to the job service if i := strings.Index(task, "/TaskService/Tasks/"); i >= 0 { resp, err = r.call(ctx, "GET", "/redfish/v1/JobService/Jobs/"+path.Base(task), nil, "") } if e := asRFError(err); e != nil && e.code == http.StatusNotFound { return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil } } if err != nil { return nil, err } var t obj _ = json.Unmarshal(resp.body, &t) st := str(t["TaskState"]) + str(t["JobState"]) res := &fwPollResult{State: "running", Pct: -1} if p, ok := t["PercentComplete"].(float64); ok { res.Pct = int(p) } for _, v := range asObj(t["Oem"]) { // iBMC TaskPercentage "40%", iRMC TotalProgressPercent o := asObj(v) if p := strings.TrimSuffix(str(o["TaskPercentage"]), "%"); p != "" { res.Pct, _ = strconv.Atoi(p) } if p, ok := o["TotalProgressPercent"].(float64); ok { res.Pct = int(p) } } if ms := toList(t["Messages"]); len(ms) > 0 { res.Msg = str(asObj(ms[len(ms)-1])["Message"]) } if res.Msg == "" { res.Msg = str(t["Message"]) // Dell jobs } if res.Pct < 0 { // no percentage on the task: some keep it on the update service res.Pct = 0 var us obj if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err == nil { if h := asObj(dig(us, "Oem", "Hpe")); len(h) > 0 { // iLO: State, FlashProgressPercent, Result if p, ok := h["FlashProgressPercent"].(float64); ok { res.Pct = int(p) } if str(h["State"]) == "Error" { res.State, res.Msg = "failed", "iLO update service: "+oneLine(fmt.Sprint(h["Result"])) return res, nil } } if a := asObj(dig(us, "Oem", "AMIUpdateService")); len(a) > 0 { // "3% done." if m := regexp.MustCompile(`(\d+)%`).FindStringSubmatch(str(a["FlashPercentage"])); m != nil { res.Pct, _ = strconv.Atoi(m[1]) } if res.Msg == "" { res.Msg = str(a["UpdateStatus"]) } } } } switch st { case "Completed": res.State, res.Pct = "done", 100 case "Exception", "Killed", "Cancelled", "Failed", "CompletedWithErrors", "Interrupted": res.State = "failed" case "Scheduled", "UserIntervention": // Dell: staged until the host reboots res.State = "done" case "": if resp.code == http.StatusOK && t["TaskState"] == nil && t["JobState"] == nil { res.State = "done" // a task monitor answers with the result once finished } } if res.State == "running" && res.Pct >= 100 && fwScheduledRe.MatchString(res.Msg) { res.State = "done" // staged for the next reboot } return res, nil }