rollcall/cmd/rollcall/rackupdate.go

package main

// Rack controllers follow the region's version. The region names the binary
// it wants (version, architecture, SHA-256) in each rack's state and serves it
// over the rack's mutual-TLS channel. The rack (unprivileged) downloads it into
// its update directory, checks the checksum and that the binary reports the
// expected version, and leaves a "ready" marker. A root path unit
// (rollcall-rack-update.path) installs it, restarts the rack and puts the
// previous binary back if the rack does not register again within a minute.
//
// The rack process is not trusted with root: it faces the provisioning
// networks. The region signs each binary with its CA key, and the root unit
// installs nothing that does not verify against the CA certificate written at
// join (/etc/rollcall/rack/ca.pem, root's).

import (
	"context"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

type rackUpdate struct {
	Version string `json:"version"`
	Arch    string `json:"arch"`
	SHA256  string `json:"sha256"`
	Sig     string `json:"sig"` // base64 DER ECDSA over the binary's SHA-256, by the region CA
}

// ---------------------------------------------------------------- region

type binCache struct {
	mu   sync.Mutex
	key  map[string]string // path -> "size/mtime"
	sums map[string]string // path -> sha256
	sigs map[string]string // sha256 -> signature (kept: a new one each time would change the state)
}

var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}, sigs: map[string]string{}}

// binarySig signs a binary's SHA-256 with the region CA key.
func (a *App) binarySig(sum string) string {
	digest, err := hex.DecodeString(sum)
	if err != nil || a.ca == nil {
		return ""
	}
	rackBins.mu.Lock()
	defer rackBins.mu.Unlock()
	if s, ok := rackBins.sigs[sum]; ok {
		return s
	}
	der, err := ecdsa.SignASN1(rand.Reader, a.ca.Key, digest)
	if err != nil {
		return ""
	}
	rackBins.sigs[sum] = base64.StdEncoding.EncodeToString(der)
	return rackBins.sigs[sum]
}

// rackBinary: the binary a rack of this architecture should run: the region's
// own for the same architecture, else rack_binaries/rollcall-linux-<arch>.
func (a *App) rackBinary(arch string) (path, sum string, ok bool) {
	switch arch { // a file name part: never a path (the rack chooses it)
	case "amd64", "arm64", "386", "arm", "riscv64", "ppc64le", "s390x", "loong64":
	default:
		return "", "", false
	}
	if arch == runtime.GOARCH && runtime.GOOS == "linux" {
		p, err := os.Executable()
		if err != nil {
			return "", "", false
		}
		path = p
	} else if a.cfg.RackBinaries != "" {
		path = filepath.Join(a.cfg.RackBinaries, "rollcall-linux-"+arch)
	} else {
		return "", "", false
	}
	st, err := os.Stat(path)
	if err != nil {
		return "", "", false
	}
	k := fmt.Sprintf("%d/%d", st.Size(), st.ModTime().UnixNano())
	rackBins.mu.Lock()
	defer rackBins.mu.Unlock()
	if rackBins.key[path] == k {
		return path, rackBins.sums[path], true
	}
	f, err := os.Open(path)
	if err != nil {
		return "", "", false
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", "", false
	}
	rackBins.key[path], rackBins.sums[path] = k, hex.EncodeToString(h.Sum(nil))
	return path, rackBins.sums[path], true
}

// rackUpdateFor: what the rack should run, nil when nothing to offer.
func (a *App) rackUpdateFor(rc RackController) *rackUpdate {
	if rc.Kind != "remote" || version == "dev" || strings.HasSuffix(version, "-dirty") {
		return nil
	}
	if _, sum, ok := a.rackBinary(rc.Arch); ok {
		return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum, Sig: a.binarySig(sum)}
	}
	return nil
}

func (a *App) handleRackBinary(w http.ResponseWriter, r *http.Request, rack string) {
	arch := r.URL.Query().Get("arch")
	path, sum, ok := a.rackBinary(arch)
	if !ok {
		fail(w, 404, fmt.Errorf("no rollcall binary for linux/%s here (set rack_binaries)", arch))
		return
	}
	f, err := os.Open(path)
	if err != nil {
		fail(w, 500, err)
		return
	}
	defer f.Close()
	st, _ := f.Stat()
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("X-Rollcall-Version", version)
	w.Header().Set("X-Rollcall-SHA256", sum)
	w.Header().Set("Content-Length", fmt.Sprint(st.Size()))
	log.Printf("rack controller %s: sending rollcall %s (linux/%s)", rack, version, arch)
	_, _ = io.Copy(w, f)
}

// ---------------------------------------------------------------- rack

type rackUpdater struct {
	mu    sync.Mutex
	tried map[string]time.Time // version -> last attempt
}

func (r *rackd) updateDir() string {
	if r.cfg.UpdateDir != "" {
		return r.cfg.UpdateDir
	}
	return "/var/lib/rollcall/update"
}

func (r *rackd) autoUpdate() bool { return r.cfg.AutoUpdate == nil || *r.cfg.AutoUpdate }

// maybeUpdate fetches and stages the region's binary when it differs from ours.
func (r *rackd) maybeUpdate(up *rackUpdate) {
	if up == nil || up.Version == "" || up.Version == version || !r.autoUpdate() || up.Arch != runtime.GOARCH {
		return
	}
	if _, err := hex.DecodeString(up.SHA256); err != nil || len(up.SHA256) != 64 || up.Sig == "" {
		log.Printf("update to %s: the region sent no valid checksum and signature", up.Version)
		return
	}
	r.upd.mu.Lock()
	if t, ok := r.upd.tried[up.Version]; ok && time.Since(t) < time.Hour {
		r.upd.mu.Unlock()
		return
	}
	r.upd.tried[up.Version] = time.Now()
	r.upd.mu.Unlock()
	dir := r.updateDir()
	if b, err := os.ReadFile(filepath.Join(dir, "last-result")); err == nil &&
		strings.Contains(string(b), up.Version) && strings.Contains(string(b), "rolled back") {
		if st, err := os.Stat(filepath.Join(dir, "last-result")); err == nil && time.Since(st.ModTime()) < 6*time.Hour {
			log.Printf("update to %s was rolled back %s ago; not retrying yet", up.Version, time.Since(st.ModTime()).Round(time.Minute))
			return
		}
	}
	if err := r.stageUpdate(up); err != nil {
		log.Printf("update to %s: %v", up.Version, err)
		return
	}
	log.Printf("update to %s staged in %s; rollcall-rack-update installs it", up.Version, dir)
}

func (r *rackd) stageUpdate(up *rackUpdate) error {
	dir := r.updateDir()
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return err
	}
	tmp := filepath.Join(dir, "rollcall.new.tmp")
	defer os.Remove(tmp)
	req, err := r.req(context.Background(), "GET", "/api/v1/rack/binary?arch="+up.Arch, nil)
	if err != nil {
		return err
	}
	resp, err := r.big.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return fmt.Errorf("download: %s %s", resp.Status, strings.TrimSpace(string(b)))
	}
	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755)
	if err != nil {
		return err
	}
	h := sha256.New()
	_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
	if cerr := f.Close(); err == nil {
		err = cerr
	}
	if err != nil {
		return err
	}
	if got := hex.EncodeToString(h.Sum(nil)); got != up.SHA256 {
		return fmt.Errorf("checksum %s, region announced %s", got[:12], up.SHA256[:12])
	}
	out, err := exec.Command(tmp, "version").Output()
	if err != nil {
		return fmt.Errorf("new binary does not run: %w", err)
	}
	if f := strings.Fields(string(out)); len(f) < 2 || f[1] != up.Version {
		return fmt.Errorf("new binary says %q, expected %s", strings.TrimSpace(string(out)), up.Version)
	}
	if err := os.Rename(tmp, filepath.Join(dir, "rollcall.new")); err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sha256"), []byte(up.SHA256+"  rollcall.new\n"), 0o644); err != nil {
		return err
	}
	sig, err := base64.StdEncoding.DecodeString(up.Sig)
	if err != nil {
		return fmt.Errorf("signature: %w", err)
	}
	if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sig"), sig, 0o644); err != nil {
		return err
	}
	ready := filepath.Join(dir, "ready")
	if err := os.WriteFile(ready+".tmp", []byte(up.Version+"\n"), 0o644); err != nil {
		return err
	}
	return os.Rename(ready+".tmp", ready)
}