diff --git a/CHANGELOG.md b/CHANGELOG.md index f8cec6c..cc11eeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,10 @@ publish a tag that has no section here. DOMAIN,CIDR [-csr FILE]` signs a name-constrained intermediate (for an ACME server such as Caddy's `acme_server`); `rollcall ca rotate -yes` replaces a root that cannot delegate; tag fragment `deploy/tags/acme-cert.yaml` fetches and renews a node certificate with certbot. +- Rack controllers update themselves to the region's version: checksum- and version-checked + download over mutual TLS, installed by a root path unit with automatic rollback; + `rack_binaries` for other architectures, `auto_update` to opt out; versions in + `rollcall controllers` and the web UI. - `rollcall version`; CD-built binary carries `git describe`. - Release notes from `CHANGELOG.md`. diff --git a/README.md b/README.md index e942e6f..0d2ed4e 100644 --- a/README.md +++ b/README.md @@ -420,6 +420,16 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and call is mutual TLS and the region identifies the rack by its certificate; `rollcall controller rm` revokes it at once. Racks renew a month before expiry. The old `rack_secret` is still accepted when configured. +* **Updates.** Racks follow the region's version: the region names the + binary (version, architecture, SHA-256) in each rack's state and serves it + over the rack's mutual-TLS channel. The rack stages it in + `/var/lib/rollcall/update` after checking the checksum and that it reports + the expected version; `rollcall-rack-update.path` (root) installs it, + restarts the rack and puts the previous binary back if the rack does not + register again within a minute (`last-result` says what happened). Racks on + another architecture get `rack_binaries/rollcall-linux-`; + `"auto_update": false` in `rack.json` opts out. `rollcall controllers` + shows each rack's version. * **Clients.** On the region host the CLI reads the CA from `ca_dir`; elsewhere `ROLLCALL_CA=ca.pem`, or the console shows the CA fingerprint on first connect and keeps the CA in `~/.config/rollcall/client.json` once you diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index ad6316b..848851b 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -38,6 +38,8 @@ type Config struct { // remote rack controllers (rollcall rack) authenticate with RackSecret RackName string `json:"rack_name"` RackSecret string `json:"rack_secret"` // legacy shared secret; rack controllers join with a code and mutual TLS now + // rollcall-linux- for rack controllers on another architecture than the region + RackBinaries string `json:"rack_binaries"` // certificate authority and HTTPS CADir string `json:"ca_dir"` // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca) diff --git a/cmd/rollcall/netapi.go b/cmd/rollcall/netapi.go index bc1d8f8..ee9dd8e 100644 --- a/cmd/rollcall/netapi.go +++ b/cmd/rollcall/netapi.go @@ -774,13 +774,13 @@ func (c *client) netCmd(cmd string, args []string) error { if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil { return err } - fmt.Printf("%-14s %-9s %-7s %-24s %8s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "LAST SEEN", "INTERFACES") + fmt.Printf("%-14s %-9s %-7s %-24s %8s %-22s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "VERSION", "LAST SEEN", "INTERFACES") for _, r := range rs { var ifs []string for _, i := range r.Interfaces { ifs = append(ifs, i.Name+"="+strings.Join(i.Addrs, ",")) } - fmt.Printf("%-14s %-9s %-7v %-24s %8d %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, timeStr(r.LastSeen), strings.Join(ifs, " ")) + fmt.Printf("%-14s %-9s %-7v %-24s %8d %-22s %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, orDash(r.Version), timeStr(r.LastSeen), strings.Join(ifs, " ")) } } case "dhcp": diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index b8687ee..c94b15a 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -23,6 +23,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -41,6 +42,7 @@ type RackController struct { URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1 Kind string `json:"kind"` // embedded | remote Version string `json:"version"` + Arch string `json:"arch,omitempty"` // GOARCH, for self-update Interfaces []RackIface `json:"interfaces"` DHCPInterfaces []string `json:"dhcp_interfaces"` TFTPRoot string `json:"tftp_root"` @@ -58,7 +60,8 @@ type rackState struct { DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here HostsDHCP string `json:"hosts_dhcp"` Hosts string `json:"hosts"` - BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP + BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP + Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run } type rackJob struct { @@ -84,7 +87,7 @@ type rackResult struct { // ---------------------------------------------------------------- store func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { - rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) if err != nil { return nil, err } @@ -93,7 +96,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { for rows.Next() { var r RackController var ifs, dif []byte - if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { + if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { return nil, err } _ = json.Unmarshal(ifs, &r.Interfaces) @@ -112,9 +115,9 @@ func (s *Store) PutRackController(ctx context.Context, r RackController) error { } ifs, _ := json.Marshal(r.Interfaces) dif, _ := json.Marshal(r.DHCPInterfaces) - _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen) VALUES ($1,$2,$3,$4,$5,$6,$7,now()) - ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now()`, - r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot) + _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,'')) + ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`, + r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch) return err } @@ -436,6 +439,9 @@ LABEL local func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { gen := a.hub.current() st := &rackState{Gen: gen} + if rc, ok := a.racksSnapshot()[name]; ok { + st.Update = a.rackUpdateFor(rc) + } base := a.rackURL(name) app := a.appendFor(base) st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app) @@ -656,7 +662,7 @@ func localInterfaces() []RackIface { // startRacks registers the embedded rack controller, carries over what an // older version kept on disk or in config, and starts the local apply loop. func (a *App) startRacks(ctx context.Context) error { - if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, + if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH, Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil { return err } diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 88e2a37..c7491b6 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -30,6 +30,7 @@ import ( "os/user" "path" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -50,6 +51,8 @@ type RackConfig struct { CAFile string `json:"ca_file,omitempty"` // set by rollcall rack join: the region's CA, CertFile string `json:"cert_file,omitempty"` // this rack controller's certificate KeyFile string `json:"key_file,omitempty"` // and key; with them the secret is not needed + AutoUpdate *bool `json:"auto_update,omitempty"` // follow the region's version (default true) + UpdateDir string `json:"update_dir,omitempty"` // default /var/lib/rollcall/update } type rackd struct { @@ -61,6 +64,7 @@ type rackd struct { gen int64 tlsc *tls.Config certMu sync.Mutex + upd rackUpdater cert *tls.Certificate headMu sync.Mutex @@ -314,7 +318,7 @@ func runRack(args []string) error { if err != nil { return fmt.Errorf("region: %w", err) } - r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}} + r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}, upd: rackUpdater{tried: map[string]time.Time{}}} if err := r.clients(); err != nil { return fmt.Errorf("tls: %w", err) } @@ -403,7 +407,7 @@ func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, erro } func (r *rackd) register() error { - _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, + _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH, Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil) return err } @@ -439,6 +443,7 @@ func (r *rackd) loop() { time.Sleep(3 * time.Second) continue } + go r.maybeUpdate(st.Update) if st.Gen == r.gen { continue } diff --git a/cmd/rollcall/rackupdate.go b/cmd/rollcall/rackupdate.go new file mode 100644 index 0000000..c89a3ea --- /dev/null +++ b/cmd/rollcall/rackupdate.go @@ -0,0 +1,213 @@ +package main + +// Rack controllers follow the region's version. The region names the binary +// it wants (version, architecture, SHA-256) in each rack's state and serves it +// over the rack's mutual-TLS channel. The rack (unprivileged) downloads it into +// its update directory, checks the checksum and that the binary reports the +// expected version, and leaves a "ready" marker. A root path unit +// (rollcall-rack-update.path) installs it, restarts the rack and puts the +// previous binary back if the rack does not register again within a minute. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "time" +) + +type rackUpdate struct { + Version string `json:"version"` + Arch string `json:"arch"` + SHA256 string `json:"sha256"` +} + +// ---------------------------------------------------------------- region + +type binCache struct { + mu sync.Mutex + key map[string]string // path -> "size/mtime" + sums map[string]string // path -> sha256 +} + +var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}} + +// rackBinary: the binary a rack of this architecture should run: the region's +// own for the same architecture, else rack_binaries/rollcall-linux-. +func (a *App) rackBinary(arch string) (path, sum string, ok bool) { + if arch == "" { + return "", "", false + } + if arch == runtime.GOARCH && runtime.GOOS == "linux" { + p, err := os.Executable() + if err != nil { + return "", "", false + } + path = p + } else if a.cfg.RackBinaries != "" { + path = filepath.Join(a.cfg.RackBinaries, "rollcall-linux-"+arch) + } else { + return "", "", false + } + st, err := os.Stat(path) + if err != nil { + return "", "", false + } + k := fmt.Sprintf("%d/%d", st.Size(), st.ModTime().UnixNano()) + rackBins.mu.Lock() + defer rackBins.mu.Unlock() + if rackBins.key[path] == k { + return path, rackBins.sums[path], true + } + f, err := os.Open(path) + if err != nil { + return "", "", false + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", "", false + } + rackBins.key[path], rackBins.sums[path] = k, hex.EncodeToString(h.Sum(nil)) + return path, rackBins.sums[path], true +} + +// rackUpdateFor: what the rack should run, nil when nothing to offer. +func (a *App) rackUpdateFor(rc RackController) *rackUpdate { + if rc.Kind != "remote" || version == "dev" || strings.HasSuffix(version, "-dirty") { + return nil + } + if _, sum, ok := a.rackBinary(rc.Arch); ok { + return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum} + } + return nil +} + +func (a *App) handleRackBinary(w http.ResponseWriter, r *http.Request, rack string) { + arch := r.URL.Query().Get("arch") + path, sum, ok := a.rackBinary(arch) + if !ok { + fail(w, 404, fmt.Errorf("no rollcall binary for linux/%s here (set rack_binaries)", arch)) + return + } + f, err := os.Open(path) + if err != nil { + fail(w, 500, err) + return + } + defer f.Close() + st, _ := f.Stat() + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("X-Rollcall-Version", version) + w.Header().Set("X-Rollcall-SHA256", sum) + w.Header().Set("Content-Length", fmt.Sprint(st.Size())) + log.Printf("rack controller %s: sending rollcall %s (linux/%s)", rack, version, arch) + _, _ = io.Copy(w, f) +} + +// ---------------------------------------------------------------- rack + +type rackUpdater struct { + mu sync.Mutex + tried map[string]time.Time // version -> last attempt +} + +func (r *rackd) updateDir() string { + if r.cfg.UpdateDir != "" { + return r.cfg.UpdateDir + } + return "/var/lib/rollcall/update" +} + +func (r *rackd) autoUpdate() bool { return r.cfg.AutoUpdate == nil || *r.cfg.AutoUpdate } + +// maybeUpdate fetches and stages the region's binary when it differs from ours. +func (r *rackd) maybeUpdate(up *rackUpdate) { + if up == nil || up.Version == "" || up.Version == version || !r.autoUpdate() || up.Arch != runtime.GOARCH { + return + } + r.upd.mu.Lock() + if t, ok := r.upd.tried[up.Version]; ok && time.Since(t) < time.Hour { + r.upd.mu.Unlock() + return + } + r.upd.tried[up.Version] = time.Now() + r.upd.mu.Unlock() + dir := r.updateDir() + if b, err := os.ReadFile(filepath.Join(dir, "last-result")); err == nil && + strings.Contains(string(b), up.Version) && strings.Contains(string(b), "rolled back") { + if st, err := os.Stat(filepath.Join(dir, "last-result")); err == nil && time.Since(st.ModTime()) < 6*time.Hour { + log.Printf("update to %s was rolled back %s ago; not retrying yet", up.Version, time.Since(st.ModTime()).Round(time.Minute)) + return + } + } + if err := r.stageUpdate(up); err != nil { + log.Printf("update to %s: %v", up.Version, err) + return + } + log.Printf("update to %s staged in %s; rollcall-rack-update installs it", up.Version, dir) +} + +func (r *rackd) stageUpdate(up *rackUpdate) error { + dir := r.updateDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tmp := filepath.Join(dir, "rollcall.new.tmp") + defer os.Remove(tmp) + req, err := r.req(context.Background(), "GET", "/api/v1/rack/binary?arch="+up.Arch, nil) + if err != nil { + return err + } + resp, err := r.big.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("download: %s %s", resp.Status, strings.TrimSpace(string(b))) + } + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755) + if err != nil { + return err + } + h := sha256.New() + _, err = io.Copy(io.MultiWriter(f, h), resp.Body) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + if got := hex.EncodeToString(h.Sum(nil)); got != up.SHA256 { + return fmt.Errorf("checksum %s, region announced %s", got[:12], up.SHA256[:12]) + } + out, err := exec.Command(tmp, "version").Output() + if err != nil { + return fmt.Errorf("new binary does not run: %w", err) + } + if f := strings.Fields(string(out)); len(f) < 2 || f[1] != up.Version { + return fmt.Errorf("new binary says %q, expected %s", strings.TrimSpace(string(out)), up.Version) + } + if err := os.Rename(tmp, filepath.Join(dir, "rollcall.new")); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sha256"), []byte(up.SHA256+" rollcall.new\n"), 0o644); err != nil { + return err + } + ready := filepath.Join(dir, "ready") + if err := os.WriteFile(ready+".tmp", []byte(up.Version+"\n"), 0o644); err != nil { + return err + } + return os.Rename(ready+".tmp", ready) +} diff --git a/cmd/rollcall/rackupdate_test.go b/cmd/rollcall/rackupdate_test.go new file mode 100644 index 0000000..a0469b0 --- /dev/null +++ b/cmd/rollcall/rackupdate_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestRackStageUpdate(t *testing.T) { + bin := []byte("#!/bin/sh\necho rollcall v9.9.9\n") + sum := sha256.Sum256(bin) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/rack/binary" || r.Header.Get("X-Rollcall-Rack") != "r2" { + http.NotFound(w, r) + return + } + _, _ = w.Write(bin) + })) + defer srv.Close() + dir := t.TempDir() + r := &rackd{cfg: RackConfig{Name: "r2", Region: srv.URL, UpdateDir: dir}, big: srv.Client(), upd: rackUpdater{tried: map[string]time.Time{}}} + good := &rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: hex.EncodeToString(sum[:])} + + if err := r.stageUpdate(&rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: "00" + good.SHA256[2:]}); err == nil { + t.Fatal("checksum mismatch accepted") + } + if err := r.stageUpdate(&rackUpdate{Version: "v1.0.0", Arch: runtime.GOARCH, SHA256: good.SHA256}); err == nil { + t.Fatal("version mismatch accepted") + } + if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil { + t.Fatal("ready marker after a rejected update") + } + if err := r.stageUpdate(good); err != nil { + t.Fatal(err) + } + for _, f := range []string{"rollcall.new", "rollcall.new.sha256", "ready"} { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Errorf("%s missing: %v", f, err) + } + } + if b, _ := os.ReadFile(filepath.Join(dir, "ready")); string(b) != "v9.9.9\n" { + t.Errorf("ready: %q", b) + } + // a rolled-back version is not fetched again right away + _ = os.Remove(filepath.Join(dir, "ready")) + _ = os.WriteFile(filepath.Join(dir, "last-result"), []byte("2026-10-28 v9.9.9 did not register within 60s; rolled back\n"), 0o644) + r.maybeUpdate(good) + if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil { + t.Error("retried a rolled-back version") + } +} diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 5637999..8b15861 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -217,6 +217,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState)) mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork)) mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult)) + mux.HandleFunc("GET /api/v1/rack/binary", a.rackAuth(a.handleRackBinary)) // physical location mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation)) mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport)) diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index af1aa91..8515d89 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -165,6 +165,7 @@ CREATE TABLE IF NOT EXISTS rack_controllers ( ); ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text; CREATE TABLE IF NOT EXISTS join_tokens ( hash text PRIMARY KEY, name text, diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index 87aeefc..4571c15 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -686,7 +686,9 @@ function wireNetPlan(id) { // ---------------------------------------------------------------- network page function pageNetwork() { - const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`${esc(c.name)}`, esc(c.kind), c.online ? 'online' : 'offline', `${esc(c.url)}`, c.machines, + const regionVer = (S.controllers.find(c => c.kind === 'embedded') || {}).version; + const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Version', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`${esc(c.name)}`, esc(c.kind), c.online ? 'online' : 'offline', `${esc(c.url)}`, c.machines, + `${esc(c.version || '—')}`, `${ago(c.last_seen)}`, `${esc((c.interfaces || []).map(i => i.name + ' ' + (i.addrs || []).join(',')).join(' · '))}`, `${c.kind === 'remote' ? ` ` : ''}`])); const sub = table(['Subnet', 'Purpose', 'Fabric / VLAN', 'DHCP', 'Gateway', 'DNS', 'Usage', 'Ranges', ''], S.subnets.map(s => { const pct = s.size ? Math.round(s.used * 100 / s.size) : 0; diff --git a/deploy/install-rack.sh b/deploy/install-rack.sh index 1956ffd..bf4a0a6 100644 --- a/deploy/install-rack.sh +++ b/deploy/install-rack.sh @@ -50,10 +50,13 @@ for f in hosts hosts.dhcp; do [ -e "$DNS/$f" ] || install -m 0644 -o rollcall -g # until the region assigns a VLAN only loopback is served printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=%s/conf.d/,*.conf\n' "$DNS" > /etc/dnsmasq.d/rollcall.conf install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh -install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service /etc/systemd/system/ +install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh +install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update +install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/ systemctl daemon-reload -systemctl enable dnsmasq.service rollcall-dnsmasq.path >/dev/null 2>&1 +systemctl enable dnsmasq.service rollcall-dnsmasq.path rollcall-rack-update.path >/dev/null 2>&1 +systemctl start rollcall-rack-update.path systemctl restart dnsmasq.service || { journalctl -u dnsmasq -n 5 --no-pager >&2; exit 1; } systemctl start rollcall-dnsmasq.path systemctl enable --now rollcall-rack.service diff --git a/deploy/rollcall-rack-update.path b/deploy/rollcall-rack-update.path new file mode 100644 index 0000000..1447554 --- /dev/null +++ b/deploy/rollcall-rack-update.path @@ -0,0 +1,8 @@ +[Unit] +Description=rollcall: install a rollcall binary the rack controller staged + +[Path] +PathExists=/var/lib/rollcall/update/ready + +[Install] +WantedBy=multi-user.target diff --git a/deploy/rollcall-rack-update.service b/deploy/rollcall-rack-update.service new file mode 100644 index 0000000..142490d --- /dev/null +++ b/deploy/rollcall-rack-update.service @@ -0,0 +1,6 @@ +[Unit] +Description=rollcall: install the staged rollcall binary, restart the rack controller, roll back on failure + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/rollcall-rack-update.sh diff --git a/deploy/rollcall-rack-update.sh b/deploy/rollcall-rack-update.sh new file mode 100755 index 0000000..a211200 --- /dev/null +++ b/deploy/rollcall-rack-update.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path). +# The rack checked the region's checksum and the binary's version; this checks +# them again, keeps the running binary as .prev, restarts the rack and puts the +# previous binary back if the rack does not register with the region in 60 s. +set -u +D=/var/lib/rollcall/update +BIN=/usr/local/bin/rollcall +[ -f "$D/ready" ] || exit 0 +want=$(cat "$D/ready"); rm -f "$D/ready" +cd "$D" || exit 1 +result() { echo "$(date -Is) $*" > "$D/last-result"; echo "$*"; } +sha256sum -c rollcall.new.sha256 >/dev/null 2>&1 || { result "$want: checksum mismatch, not installed"; exit 1; } +got=$(./rollcall.new version 2>/dev/null | awk '{print $2}') +[ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; } +cur=$("$BIN" version 2>/dev/null | awk '{print $2}') +cp -p "$BIN" "$BIN.prev" +install -m 0755 rollcall.new "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN" +since=$(date +%s) +systemctl restart rollcall-rack.service +for _ in $(seq 1 30); do + sleep 2 + if journalctl -u rollcall-rack.service --since "@$since" -o cat 2>/dev/null | grep -q "^.*registered with"; then + rm -f rollcall.new rollcall.new.sha256 + result "updated $cur -> $want" + exit 0 + fi +done +cp -p "$BIN.prev" "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN" +systemctl restart rollcall-rack.service +result "$want did not register within 60s; rolled back to $cur" +exit 1