Rack certificate renewal: rack-owned state, atomic, grace period

Renewal could not work: rack join writes the certificate and key as
root, and the rack (user rollcall) failed to overwrite them. A lost
renewal answer also locked the rack out, since the region accepted only
the new serial.

- the renewed certificate and key go into one file in
  /var/lib/rollcall/rack-cert (cert_state_dir), written atomically, and
  are preferred at start when they are this rack's, from the CA, newer
- after a renewal the region accepts the previous serial for 7 days; a
  new join does not keep it
- rollcall rack renew renews now (run as the service user)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/rollcall/ca.go

@@ -427,15 +427,30 @@ func (s *Store) UseJoinToken(ctx context.Context, hash string) (string, error) {
 	return name, nil
 }
 
-func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time) error {
-	_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires)
+// rackCertGrace: after a renewal the previous certificate still works this
+// long, so a renewal answer lost on the way does not lock the rack out.
+const rackCertGrace = 7 * 24 * time.Hour
+
+// SetRackCert records a rack's certificate. A renewal keeps the previous
+// serial for rackCertGrace; an enrollment (a new join) does not.
+func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time, renewal bool) error {
+	if renewal {
+		_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET prev_cert_serial=cert_serial, prev_cert_until=now()+$4::interval,
+			cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires, fmt.Sprintf("%d seconds", int(rackCertGrace.Seconds())))
+		return err
+	}
+	_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3, prev_cert_serial=NULL, prev_cert_until=NULL WHERE name=$1`, name, serial, expires)
 	return err
 }
 
-func (s *Store) RackCertSerial(ctx context.Context, name string) string {
-	var serial string
-	_ = s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,'') FROM rack_controllers WHERE name=$1`, name).Scan(&serial)
-	return serial
+// rackCertOK: the serial is the rack's current certificate, or the previous one within the grace period.
+func (s *Store) rackCertOK(ctx context.Context, name, serial string) bool {
+	var cur, prev string
+	var until *time.Time
+	if err := s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,''), coalesce(prev_cert_serial,''), prev_cert_until FROM rack_controllers WHERE name=$1`, name).Scan(&cur, &prev, &until); err != nil {
+		return false
+	}
+	return serial != "" && (serial == cur || (serial == prev && until != nil && time.Now().Before(*until)))
 }
 
 // ---------------------------------------------------------------- HTTP
@@ -451,7 +466,7 @@ func (a *App) peerRack(r *http.Request) string {
 	if name == c.Subject.CommonName { // only rack certificates
 		return ""
 	}
-	if a.store.RackCertSerial(r.Context(), name) != c.SerialNumber.Text(16) {
+	if !a.store.rackCertOK(r.Context(), name, c.SerialNumber.Text(16)) {
 		return ""
 	}
 	return name
@@ -581,7 +596,7 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) {
 		fail(w, 500, err)
 		return
 	}
-	if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
+	if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter, false); err != nil {
 		fail(w, 500, err)
 		return
 	}
@@ -607,7 +622,7 @@ func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack strin
 		fail(w, 400, err)
 		return
 	}
-	if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
+	if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter, true); err != nil {
 		fail(w, 500, err)
 		return
 	}

cmd/rollcall/ca_test.go

@@ -9,6 +9,8 @@ import (
 	"crypto/x509"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -262,3 +264,50 @@ func TestAdminClientCertificate(t *testing.T) {
 		t.Error("token no longer works")
 	}
 }
+
+// A renewed certificate is kept in the rack's own directory and preferred
+// when it is this rack's, from the CA and newer.
+func TestRackRenewedCertificate(t *testing.T) {
+	ca, _ := newCA("test CA")
+	a := &App{ca: ca, cfg: &Config{}}
+	pool := x509.NewCertPool()
+	pool.AddCert(ca.Cert)
+	issue := func(name string) ([]byte, []byte, *x509.Certificate) {
+		key, csr, _ := newCSR("rack:" + name)
+		certPEM, cert, err := a.signRackCSR(csr, name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return certPEM, key, cert
+	}
+	_, _, joined := issue("hall-b")
+	dir := t.TempDir()
+	p := filepath.Join(dir, "rack.pem")
+	if loadRenewed(p, pool, "hall-b", joined) != nil {
+		t.Error("nothing renewed yet")
+	}
+	time.Sleep(1100 * time.Millisecond) // a later NotAfter
+	certPEM, key, renewed := issue("hall-b")
+	if err := writeAtomic(p, append(certPEM, key...), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	got := loadRenewed(p, pool, "hall-b", joined)
+	if got == nil || got.Leaf.SerialNumber.Cmp(renewed.SerialNumber) != 0 {
+		t.Fatalf("renewed certificate not used: %v", got)
+	}
+	if loadRenewed(p, pool, "hall-c", joined) != nil {
+		t.Error("another rack's certificate was used")
+	}
+	if loadRenewed(p, pool, "hall-b", renewed) != nil {
+		t.Error("an older or equal certificate replaced the current one")
+	}
+	other, _ := newCA("other CA")
+	op := x509.NewCertPool()
+	op.AddCert(other.Cert)
+	if loadRenewed(p, op, "hall-b", joined) != nil {
+		t.Error("a certificate from another CA was used")
+	}
+	if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
+		t.Errorf("mode %v", st.Mode())
+	}
+}

cmd/rollcall/main.go

@@ -100,6 +100,13 @@ func main() {
 	cmd, args := os.Args[1], os.Args[2:]
 	switch cmd {
 	case "rack":
+		if len(args) > 0 && args[0] == "renew" {
+			if err := rackRenew(args[1:]); err != nil {
+				fmt.Fprintln(os.Stderr, "error:", err)
+				os.Exit(1)
+			}
+			return
+		}
 		if len(args) > 0 && args[0] == "join" {
 			if err := rackJoin(args[1:]); err != nil {
 				fmt.Fprintln(os.Stderr, "error:", err)

cmd/rollcall/rackd.go

@@ -54,6 +54,14 @@ type RackConfig struct {
 	KeyFile        string   `json:"key_file,omitempty"`        // and key; with them the secret is not needed
 	AutoUpdate     *bool    `json:"auto_update,omitempty"`     // follow the region's version (default true)
 	UpdateDir      string   `json:"update_dir,omitempty"`      // default /var/lib/rollcall/update
+	// CertStateDir: where a renewed certificate goes (cert and key in one file,
+	// rack.pem). The files from rack join are root's; this directory is the
+	// service user's. Default /var/lib/rollcall/rack-cert.
+	CertStateDir string `json:"cert_state_dir,omitempty"`
+}
+
+func (c RackConfig) renewedPath() string {
+	return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "rack.pem")
 }
 
 type rackd struct {
@@ -137,6 +145,9 @@ func (r *rackd) clients() error {
 		}
 		pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
 		r.cert = &pair
+		if renewed := loadRenewed(r.cfg.renewedPath(), pool, r.cfg.Name, pair.Leaf); renewed != nil {
+			r.cert = renewed
+		}
 		r.tlsc.RootCAs = pool
 		r.tlsc.VerifyConnection = regionLeaf
 		r.tlsc.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
@@ -186,11 +197,9 @@ func (r *rackd) renew() error {
 		return err
 	}
 	pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
-	if err := os.WriteFile(r.cfg.KeyFile, key, 0o640); err != nil {
-		return err
-	}
-	if err := os.WriteFile(r.cfg.CertFile, []byte(resp.Cert), 0o644); err != nil {
-		return err
+	// one file, renamed into place: a crash never leaves a key without its certificate
+	if err := writeAtomic(r.cfg.renewedPath(), append([]byte(resp.Cert), key...), 0o600); err != nil {
+		return fmt.Errorf("keep the renewed certificate (the region already switched to it; until this works the previous one is accepted for 7 days): %w; create the directory: install -d -m 0700 -o rollcall -g rollcall %s", err, filepath.Dir(r.cfg.renewedPath()))
 	}
 	r.certMu.Lock()
 	r.cert = &pair
@@ -199,6 +208,84 @@ func (r *rackd) renew() error {
 	return nil
 }
 
+// loadRenewed is the certificate renew() kept, when it belongs to this rack,
+// comes from the CA and outlives the one from rack join.
+func loadRenewed(p string, pool *x509.CertPool, name string, current *x509.Certificate) *tls.Certificate {
+	b, err := os.ReadFile(p)
+	if err != nil {
+		return nil
+	}
+	pair, err := tls.X509KeyPair(b, b)
+	if err != nil {
+		log.Printf("%s: %v (ignored)", p, err)
+		return nil
+	}
+	leaf, err := x509.ParseCertificate(pair.Certificate[0])
+	if err != nil || leaf.Subject.CommonName != "rack:"+name {
+		return nil
+	}
+	if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
+		log.Printf("%s: %v (ignored)", p, err)
+		return nil
+	}
+	if current != nil && !leaf.NotAfter.After(current.NotAfter) {
+		return nil
+	}
+	pair.Leaf = leaf
+	return &pair
+}
+
+func writeAtomic(p string, data []byte, mode os.FileMode) error {
+	f, err := os.CreateTemp(filepath.Dir(p), ".tmp-*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(f.Name())
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Chmod(mode); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	return os.Rename(f.Name(), p)
+}
+
+// rackRenew renews the certificate now (rollcall rack renew), as the service user.
+func rackRenew(args []string) error {
+	fs := newFlags("rack renew")
+	cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration")
+	_ = fs.Parse(args)
+	cfg, err := loadRackConfig(*cfgPath)
+	if err != nil {
+		return err
+	}
+	ru, err := url.Parse(cfg.Region)
+	if err != nil {
+		return err
+	}
+	r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}}
+	if err := r.clients(); err != nil {
+		return err
+	}
+	if r.cert == nil {
+		return errors.New("this rack controller has no certificate (rollcall rack join)")
+	}
+	if err := r.renew(); err != nil {
+		return err
+	}
+	fmt.Printf("renewed: %s, valid until %s; restart rollcall-rack to use it\n", cfg.renewedPath(), r.cert.Leaf.NotAfter.Format("2006-01-02"))
+	return nil
+}
+
 func newCSR(cn string) (keyPEM []byte, csrPEM string, err error) {
 	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {

cmd/rollcall/store.go

@@ -164,6 +164,8 @@ CREATE TABLE IF NOT EXISTS rack_controllers (
   registered      timestamptz NOT NULL DEFAULT now()
 );
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text;
+ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text;
+ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text;
 ALTER TABLE vlans ADD COLUMN IF NOT EXISTS dhcp_mode text NOT NULL DEFAULT '';

deploy/install-rack.sh

@@ -53,6 +53,7 @@ install -d -m 0755 /var/lib/rollcall-dnsmasq
 install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
 install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh
 install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update
+install -d -m 0700 -o rollcall -g rollcall /var/lib/rollcall/rack-cert # renewed certificates (the joined ones are root's)
 install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/
 
 systemctl daemon-reload