Rack controllers update themselves to the region's version

The region offers its binary (version, arch, sha256) in each rack's state and
serves it over mutual TLS; the rack stages it after checking checksum and
version; a root path unit installs it, restarts the rack and rolls back if it
does not register within a minute.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -62,6 +62,10 @@ publish a tag that has no section here.
   DOMAIN,CIDR [-csr FILE]` signs a name-constrained intermediate (for an ACME server such as Caddy's
   `acme_server`); `rollcall ca rotate -yes` replaces a root that cannot delegate; tag fragment
   `deploy/tags/acme-cert.yaml` fetches and renews a node certificate with certbot.
+- Rack controllers update themselves to the region's version: checksum- and version-checked
+  download over mutual TLS, installed by a root path unit with automatic rollback;
+  `rack_binaries` for other architectures, `auto_update` to opt out; versions in
+  `rollcall controllers` and the web UI.
 - `rollcall version`; CD-built binary carries `git describe`.
 - Release notes from `CHANGELOG.md`.
 

README.md

@@ -420,6 +420,16 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and
   call is mutual TLS and the region identifies the rack by its certificate;
   `rollcall controller rm` revokes it at once. Racks renew a month before
   expiry. The old `rack_secret` is still accepted when configured.
+* **Updates.** Racks follow the region's version: the region names the
+  binary (version, architecture, SHA-256) in each rack's state and serves it
+  over the rack's mutual-TLS channel. The rack stages it in
+  `/var/lib/rollcall/update` after checking the checksum and that it reports
+  the expected version; `rollcall-rack-update.path` (root) installs it,
+  restarts the rack and puts the previous binary back if the rack does not
+  register again within a minute (`last-result` says what happened). Racks on
+  another architecture get `rack_binaries/rollcall-linux-<arch>`;
+  `"auto_update": false` in `rack.json` opts out. `rollcall controllers`
+  shows each rack's version.
 * **Clients.** On the region host the CLI reads the CA from `ca_dir`;
   elsewhere `ROLLCALL_CA=ca.pem`, or the console shows the CA fingerprint on
   first connect and keeps the CA in `~/.config/rollcall/client.json` once you

cmd/rollcall/config.go

@@ -38,6 +38,8 @@ type Config struct {
 	// remote rack controllers (rollcall rack) authenticate with RackSecret
 	RackName   string `json:"rack_name"`
 	RackSecret string `json:"rack_secret"` // legacy shared secret; rack controllers join with a code and mutual TLS now
+	// rollcall-linux-<arch> for rack controllers on another architecture than the region
+	RackBinaries string `json:"rack_binaries"`
 
 	// certificate authority and HTTPS
 	CADir     string   `json:"ca_dir"`     // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca)

cmd/rollcall/netapi.go

@@ -774,13 +774,13 @@ func (c *client) netCmd(cmd string, args []string) error {
 			if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil {
 				return err
 			}
-			fmt.Printf("%-14s %-9s %-7s %-24s %8s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "LAST SEEN", "INTERFACES")
+			fmt.Printf("%-14s %-9s %-7s %-24s %8s %-22s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "VERSION", "LAST SEEN", "INTERFACES")
 			for _, r := range rs {
 				var ifs []string
 				for _, i := range r.Interfaces {
 					ifs = append(ifs, i.Name+"="+strings.Join(i.Addrs, ","))
 				}
-				fmt.Printf("%-14s %-9s %-7v %-24s %8d %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, timeStr(r.LastSeen), strings.Join(ifs, " "))
+				fmt.Printf("%-14s %-9s %-7v %-24s %8d %-22s %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, orDash(r.Version), timeStr(r.LastSeen), strings.Join(ifs, " "))
 			}
 		}
 	case "dhcp":

cmd/rollcall/rackctl.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -41,6 +42,7 @@ type RackController struct {
 	URL            string      `json:"url"`  // how machines reach it, e.g. http://10.213.78.1
 	Kind           string      `json:"kind"` // embedded | remote
 	Version        string      `json:"version"`
+	Arch           string      `json:"arch,omitempty"` // GOARCH, for self-update
 	Interfaces     []RackIface `json:"interfaces"`
 	DHCPInterfaces []string    `json:"dhcp_interfaces"`
 	TFTPRoot       string      `json:"tftp_root"`
@@ -58,7 +60,8 @@ type rackState struct {
 	DnsmasqConf string            `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
 	HostsDHCP   string            `json:"hosts_dhcp"`
 	Hosts       string            `json:"hosts"`
-	BootFiles   []string          `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
+	BootFiles   []string          `json:"boot_files"`       // kernels and initrds the rack keeps in its cache for TFTP
+	Update      *rackUpdate       `json:"update,omitempty"` // the binary the region wants this rack to run
 }
 
 type rackJob struct {
@@ -84,7 +87,7 @@ type rackResult struct {
 // ---------------------------------------------------------------- store
 
 func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
-	rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
+	rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +96,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
 	for rows.Next() {
 		var r RackController
 		var ifs, dif []byte
-		if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
+		if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
 			return nil, err
 		}
 		_ = json.Unmarshal(ifs, &r.Interfaces)
@@ -112,9 +115,9 @@ func (s *Store) PutRackController(ctx context.Context, r RackController) error {
 	}
 	ifs, _ := json.Marshal(r.Interfaces)
 	dif, _ := json.Marshal(r.DHCPInterfaces)
-	_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen) VALUES ($1,$2,$3,$4,$5,$6,$7,now())
+	_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
-		ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now()`,
+		ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
-		r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot)
+		r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
 	return err
 }
 
@@ -436,6 +439,9 @@ LABEL local
 func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
 	gen := a.hub.current()
 	st := &rackState{Gen: gen}
+	if rc, ok := a.racksSnapshot()[name]; ok {
+		st.Update = a.rackUpdateFor(rc)
+	}
 	base := a.rackURL(name)
 	app := a.appendFor(base)
 	st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
@@ -656,7 +662,7 @@ func localInterfaces() []RackIface {
 // startRacks registers the embedded rack controller, carries over what an
 // older version kept on disk or in config, and starts the local apply loop.
 func (a *App) startRacks(ctx context.Context) error {
-	if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version,
+	if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
 		Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
 		return err
 	}

cmd/rollcall/rackd.go

@@ -30,6 +30,7 @@ import (
 	"os/user"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -50,6 +51,8 @@ type RackConfig struct {
 	CAFile         string   `json:"ca_file,omitempty"`         // set by rollcall rack join: the region's CA,
 	CertFile       string   `json:"cert_file,omitempty"`       // this rack controller's certificate
 	KeyFile        string   `json:"key_file,omitempty"`        // and key; with them the secret is not needed
+	AutoUpdate     *bool    `json:"auto_update,omitempty"`     // follow the region's version (default true)
+	UpdateDir      string   `json:"update_dir,omitempty"`      // default /var/lib/rollcall/update
 }
 
 type rackd struct {
@@ -61,6 +64,7 @@ type rackd struct {
 	gen    int64
 	tlsc   *tls.Config
 	certMu sync.Mutex
+	upd    rackUpdater
 	cert   *tls.Certificate
 
 	headMu sync.Mutex
@@ -314,7 +318,7 @@ func runRack(args []string) error {
 	if err != nil {
 		return fmt.Errorf("region: %w", err)
 	}
-	r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}}
+	r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}, upd: rackUpdater{tried: map[string]time.Time{}}}
 	if err := r.clients(); err != nil {
 		return fmt.Errorf("tls: %w", err)
 	}
@@ -403,7 +407,7 @@ func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, erro
 }
 
 func (r *rackd) register() error {
-	_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version,
+	_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH,
 		Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil)
 	return err
 }
@@ -439,6 +443,7 @@ func (r *rackd) loop() {
 			time.Sleep(3 * time.Second)
 			continue
 		}
+		go r.maybeUpdate(st.Update)
 		if st.Gen == r.gen {
 			continue
 		}

cmd/rollcall/rackupdate.go

@@ -0,0 +1,213 @@
+package main
+
+// Rack controllers follow the region's version. The region names the binary
+// it wants (version, architecture, SHA-256) in each rack's state and serves it
+// over the rack's mutual-TLS channel. The rack (unprivileged) downloads it into
+// its update directory, checks the checksum and that the binary reports the
+// expected version, and leaves a "ready" marker. A root path unit
+// (rollcall-rack-update.path) installs it, restarts the rack and puts the
+// previous binary back if the rack does not register again within a minute.
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+type rackUpdate struct {
+	Version string `json:"version"`
+	Arch    string `json:"arch"`
+	SHA256  string `json:"sha256"`
+}
+
+// ---------------------------------------------------------------- region
+
+type binCache struct {
+	mu   sync.Mutex
+	key  map[string]string // path -> "size/mtime"
+	sums map[string]string // path -> sha256
+}
+
+var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}}
+
+// rackBinary: the binary a rack of this architecture should run: the region's
+// own for the same architecture, else rack_binaries/rollcall-linux-<arch>.
+func (a *App) rackBinary(arch string) (path, sum string, ok bool) {
+	if arch == "" {
+		return "", "", false
+	}
+	if arch == runtime.GOARCH && runtime.GOOS == "linux" {
+		p, err := os.Executable()
+		if err != nil {
+			return "", "", false
+		}
+		path = p
+	} else if a.cfg.RackBinaries != "" {
+		path = filepath.Join(a.cfg.RackBinaries, "rollcall-linux-"+arch)
+	} else {
+		return "", "", false
+	}
+	st, err := os.Stat(path)
+	if err != nil {
+		return "", "", false
+	}
+	k := fmt.Sprintf("%d/%d", st.Size(), st.ModTime().UnixNano())
+	rackBins.mu.Lock()
+	defer rackBins.mu.Unlock()
+	if rackBins.key[path] == k {
+		return path, rackBins.sums[path], true
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return "", "", false
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", "", false
+	}
+	rackBins.key[path], rackBins.sums[path] = k, hex.EncodeToString(h.Sum(nil))
+	return path, rackBins.sums[path], true
+}
+
+// rackUpdateFor: what the rack should run, nil when nothing to offer.
+func (a *App) rackUpdateFor(rc RackController) *rackUpdate {
+	if rc.Kind != "remote" || version == "dev" || strings.HasSuffix(version, "-dirty") {
+		return nil
+	}
+	if _, sum, ok := a.rackBinary(rc.Arch); ok {
+		return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum}
+	}
+	return nil
+}
+
+func (a *App) handleRackBinary(w http.ResponseWriter, r *http.Request, rack string) {
+	arch := r.URL.Query().Get("arch")
+	path, sum, ok := a.rackBinary(arch)
+	if !ok {
+		fail(w, 404, fmt.Errorf("no rollcall binary for linux/%s here (set rack_binaries)", arch))
+		return
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	defer f.Close()
+	st, _ := f.Stat()
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("X-Rollcall-Version", version)
+	w.Header().Set("X-Rollcall-SHA256", sum)
+	w.Header().Set("Content-Length", fmt.Sprint(st.Size()))
+	log.Printf("rack controller %s: sending rollcall %s (linux/%s)", rack, version, arch)
+	_, _ = io.Copy(w, f)
+}
+
+// ---------------------------------------------------------------- rack
+
+type rackUpdater struct {
+	mu    sync.Mutex
+	tried map[string]time.Time // version -> last attempt
+}
+
+func (r *rackd) updateDir() string {
+	if r.cfg.UpdateDir != "" {
+		return r.cfg.UpdateDir
+	}
+	return "/var/lib/rollcall/update"
+}
+
+func (r *rackd) autoUpdate() bool { return r.cfg.AutoUpdate == nil || *r.cfg.AutoUpdate }
+
+// maybeUpdate fetches and stages the region's binary when it differs from ours.
+func (r *rackd) maybeUpdate(up *rackUpdate) {
+	if up == nil || up.Version == "" || up.Version == version || !r.autoUpdate() || up.Arch != runtime.GOARCH {
+		return
+	}
+	r.upd.mu.Lock()
+	if t, ok := r.upd.tried[up.Version]; ok && time.Since(t) < time.Hour {
+		r.upd.mu.Unlock()
+		return
+	}
+	r.upd.tried[up.Version] = time.Now()
+	r.upd.mu.Unlock()
+	dir := r.updateDir()
+	if b, err := os.ReadFile(filepath.Join(dir, "last-result")); err == nil &&
+		strings.Contains(string(b), up.Version) && strings.Contains(string(b), "rolled back") {
+		if st, err := os.Stat(filepath.Join(dir, "last-result")); err == nil && time.Since(st.ModTime()) < 6*time.Hour {
+			log.Printf("update to %s was rolled back %s ago; not retrying yet", up.Version, time.Since(st.ModTime()).Round(time.Minute))
+			return
+		}
+	}
+	if err := r.stageUpdate(up); err != nil {
+		log.Printf("update to %s: %v", up.Version, err)
+		return
+	}
+	log.Printf("update to %s staged in %s; rollcall-rack-update installs it", up.Version, dir)
+}
+
+func (r *rackd) stageUpdate(up *rackUpdate) error {
+	dir := r.updateDir()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	tmp := filepath.Join(dir, "rollcall.new.tmp")
+	defer os.Remove(tmp)
+	req, err := r.req(context.Background(), "GET", "/api/v1/rack/binary?arch="+up.Arch, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := r.big.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+		return fmt.Errorf("download: %s %s", resp.Status, strings.TrimSpace(string(b)))
+	}
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755)
+	if err != nil {
+		return err
+	}
+	h := sha256.New()
+	_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return err
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != up.SHA256 {
+		return fmt.Errorf("checksum %s, region announced %s", got[:12], up.SHA256[:12])
+	}
+	out, err := exec.Command(tmp, "version").Output()
+	if err != nil {
+		return fmt.Errorf("new binary does not run: %w", err)
+	}
+	if f := strings.Fields(string(out)); len(f) < 2 || f[1] != up.Version {
+		return fmt.Errorf("new binary says %q, expected %s", strings.TrimSpace(string(out)), up.Version)
+	}
+	if err := os.Rename(tmp, filepath.Join(dir, "rollcall.new")); err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sha256"), []byte(up.SHA256+"  rollcall.new\n"), 0o644); err != nil {
+		return err
+	}
+	ready := filepath.Join(dir, "ready")
+	if err := os.WriteFile(ready+".tmp", []byte(up.Version+"\n"), 0o644); err != nil {
+		return err
+	}
+	return os.Rename(ready+".tmp", ready)
+}

cmd/rollcall/rackupdate_test.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func TestRackStageUpdate(t *testing.T) {
+	bin := []byte("#!/bin/sh\necho rollcall v9.9.9\n")
+	sum := sha256.Sum256(bin)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v1/rack/binary" || r.Header.Get("X-Rollcall-Rack") != "r2" {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write(bin)
+	}))
+	defer srv.Close()
+	dir := t.TempDir()
+	r := &rackd{cfg: RackConfig{Name: "r2", Region: srv.URL, UpdateDir: dir}, big: srv.Client(), upd: rackUpdater{tried: map[string]time.Time{}}}
+	good := &rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: hex.EncodeToString(sum[:])}
+
+	if err := r.stageUpdate(&rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: "00" + good.SHA256[2:]}); err == nil {
+		t.Fatal("checksum mismatch accepted")
+	}
+	if err := r.stageUpdate(&rackUpdate{Version: "v1.0.0", Arch: runtime.GOARCH, SHA256: good.SHA256}); err == nil {
+		t.Fatal("version mismatch accepted")
+	}
+	if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil {
+		t.Fatal("ready marker after a rejected update")
+	}
+	if err := r.stageUpdate(good); err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range []string{"rollcall.new", "rollcall.new.sha256", "ready"} {
+		if _, err := os.Stat(filepath.Join(dir, f)); err != nil {
+			t.Errorf("%s missing: %v", f, err)
+		}
+	}
+	if b, _ := os.ReadFile(filepath.Join(dir, "ready")); string(b) != "v9.9.9\n" {
+		t.Errorf("ready: %q", b)
+	}
+	// a rolled-back version is not fetched again right away
+	_ = os.Remove(filepath.Join(dir, "ready"))
+	_ = os.WriteFile(filepath.Join(dir, "last-result"), []byte("2026-10-28 v9.9.9 did not register within 60s; rolled back\n"), 0o644)
+	r.maybeUpdate(good)
+	if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil {
+		t.Error("retried a rolled-back version")
+	}
+}

cmd/rollcall/server.go

@@ -217,6 +217,7 @@ func (a *App) routes() http.Handler {
 	mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
 	mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
 	mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
+	mux.HandleFunc("GET /api/v1/rack/binary", a.rackAuth(a.handleRackBinary))
 	// physical location
 	mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
 	mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))

cmd/rollcall/store.go

@@ -165,6 +165,7 @@ CREATE TABLE IF NOT EXISTS rack_controllers (
 );
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;
+ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text;
 CREATE TABLE IF NOT EXISTS join_tokens (
   hash    text PRIMARY KEY,
   name    text,

cmd/rollcall/web/app.js

@@ -686,7 +686,9 @@ function wireNetPlan(id) {
 
 // ---------------------------------------------------------------- network page
 function pageNetwork() {
-  const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`<b>${esc(c.name)}</b>`, esc(c.kind), c.online ? '<span class="dot ok"></span>online' : '<span class="dot bad"></span><span class="bad">offline</span>', `<span class="mono">${esc(c.url)}</span>`, c.machines,
+  const regionVer = (S.controllers.find(c => c.kind === 'embedded') || {}).version;
+  const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Version', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`<b>${esc(c.name)}</b>`, esc(c.kind), c.online ? '<span class="dot ok"></span>online' : '<span class="dot bad"></span><span class="bad">offline</span>', `<span class="mono">${esc(c.url)}</span>`, c.machines,
+    `<span class="mono${regionVer && c.version && c.version !== regionVer ? ' warn" title="differs from the region; it updates itself' : ''}">${esc(c.version || '—')}</span>`,
     `<span class="muted" title="${esc(when(c.last_seen))}">${ago(c.last_seen)}</span>`, `<span class="mono muted">${esc((c.interfaces || []).map(i => i.name + ' ' + (i.addrs || []).join(',')).join(' · '))}</span>`, `<button class="btn sm" data-dhcp="${esc(c.name)}">dnsmasq</button>${c.kind === 'remote' ? ` <button class="btn sm danger" data-rmctl="${esc(c.name)}">remove</button>` : ''}`]));
   const sub = table(['Subnet', 'Purpose', 'Fabric / VLAN', 'DHCP', 'Gateway', 'DNS', 'Usage', 'Ranges', ''], S.subnets.map(s => {
     const pct = s.size ? Math.round(s.used * 100 / s.size) : 0;

deploy/install-rack.sh

@@ -50,10 +50,13 @@ for f in hosts hosts.dhcp; do [ -e "$DNS/$f" ] || install -m 0644 -o rollcall -g
 # until the region assigns a VLAN only loopback is served
 printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=%s/conf.d/,*.conf\n' "$DNS" > /etc/dnsmasq.d/rollcall.conf
 install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
-install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service /etc/systemd/system/
+install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh
+install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update
+install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/
 
 systemctl daemon-reload
-systemctl enable dnsmasq.service rollcall-dnsmasq.path >/dev/null 2>&1
+systemctl enable dnsmasq.service rollcall-dnsmasq.path rollcall-rack-update.path >/dev/null 2>&1
+systemctl start rollcall-rack-update.path
 systemctl restart dnsmasq.service || { journalctl -u dnsmasq -n 5 --no-pager >&2; exit 1; }
 systemctl start rollcall-dnsmasq.path
 systemctl enable --now rollcall-rack.service

deploy/rollcall-rack-update.path

@@ -0,0 +1,8 @@
+[Unit]
+Description=rollcall: install a rollcall binary the rack controller staged
+
+[Path]
+PathExists=/var/lib/rollcall/update/ready
+
+[Install]
+WantedBy=multi-user.target

deploy/rollcall-rack-update.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=rollcall: install the staged rollcall binary, restart the rack controller, roll back on failure
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/rollcall-rack-update.sh

deploy/rollcall-rack-update.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path).
+# The rack checked the region's checksum and the binary's version; this checks
+# them again, keeps the running binary as .prev, restarts the rack and puts the
+# previous binary back if the rack does not register with the region in 60 s.
+set -u
+D=/var/lib/rollcall/update
+BIN=/usr/local/bin/rollcall
+[ -f "$D/ready" ] || exit 0
+want=$(cat "$D/ready"); rm -f "$D/ready"
+cd "$D" || exit 1
+result() { echo "$(date -Is) $*" > "$D/last-result"; echo "$*"; }
+sha256sum -c rollcall.new.sha256 >/dev/null 2>&1 || { result "$want: checksum mismatch, not installed"; exit 1; }
+got=$(./rollcall.new version 2>/dev/null | awk '{print $2}')
+[ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; }
+cur=$("$BIN" version 2>/dev/null | awk '{print $2}')
+cp -p "$BIN" "$BIN.prev"
+install -m 0755 rollcall.new "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
+since=$(date +%s)
+systemctl restart rollcall-rack.service
+for _ in $(seq 1 30); do
+  sleep 2
+  if journalctl -u rollcall-rack.service --since "@$since" -o cat 2>/dev/null | grep -q "^.*registered with"; then
+    rm -f rollcall.new rollcall.new.sha256
+    result "updated $cur -> $want"
+    exit 0
+  fi
+done
+cp -p "$BIN.prev" "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
+systemctl restart rollcall-rack.service
+result "$want did not register within 60s; rolled back to $cur"
+exit 1