diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index ee0b44f..f255edb 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -427,15 +427,30 @@ func (s *Store) UseJoinToken(ctx context.Context, hash string) (string, error) { return name, nil } -func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time) error { - _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires) +// rackCertGrace: after a renewal the previous certificate still works this +// long, so a renewal answer lost on the way does not lock the rack out. +const rackCertGrace = 7 * 24 * time.Hour + +// SetRackCert records a rack's certificate. A renewal keeps the previous +// serial for rackCertGrace; an enrollment (a new join) does not. +func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time, renewal bool) error { + if renewal { + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET prev_cert_serial=cert_serial, prev_cert_until=now()+$4::interval, + cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires, fmt.Sprintf("%d seconds", int(rackCertGrace.Seconds()))) + return err + } + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3, prev_cert_serial=NULL, prev_cert_until=NULL WHERE name=$1`, name, serial, expires) return err } -func (s *Store) RackCertSerial(ctx context.Context, name string) string { - var serial string - _ = s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,'') FROM rack_controllers WHERE name=$1`, name).Scan(&serial) - return serial +// rackCertOK: the serial is the rack's current certificate, or the previous one within the grace period. +func (s *Store) rackCertOK(ctx context.Context, name, serial string) bool { + var cur, prev string + var until *time.Time + if err := s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,''), coalesce(prev_cert_serial,''), prev_cert_until FROM rack_controllers WHERE name=$1`, name).Scan(&cur, &prev, &until); err != nil { + return false + } + return serial != "" && (serial == cur || (serial == prev && until != nil && time.Now().Before(*until))) } // ---------------------------------------------------------------- HTTP @@ -451,7 +466,7 @@ func (a *App) peerRack(r *http.Request) string { if name == c.Subject.CommonName { // only rack certificates return "" } - if a.store.RackCertSerial(r.Context(), name) != c.SerialNumber.Text(16) { + if !a.store.rackCertOK(r.Context(), name, c.SerialNumber.Text(16)) { return "" } return name @@ -581,7 +596,7 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) { fail(w, 500, err) return } - if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter); err != nil { + if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter, false); err != nil { fail(w, 500, err) return } @@ -607,7 +622,7 @@ func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack strin fail(w, 400, err) return } - if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter); err != nil { + if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter, true); err != nil { fail(w, 500, err) return } diff --git a/cmd/rollcall/ca_test.go b/cmd/rollcall/ca_test.go index d1b362a..b591edd 100644 --- a/cmd/rollcall/ca_test.go +++ b/cmd/rollcall/ca_test.go @@ -9,6 +9,8 @@ import ( "crypto/x509" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -262,3 +264,50 @@ func TestAdminClientCertificate(t *testing.T) { t.Error("token no longer works") } } + +// A renewed certificate is kept in the rack's own directory and preferred +// when it is this rack's, from the CA and newer. +func TestRackRenewedCertificate(t *testing.T) { + ca, _ := newCA("test CA") + a := &App{ca: ca, cfg: &Config{}} + pool := x509.NewCertPool() + pool.AddCert(ca.Cert) + issue := func(name string) ([]byte, []byte, *x509.Certificate) { + key, csr, _ := newCSR("rack:" + name) + certPEM, cert, err := a.signRackCSR(csr, name) + if err != nil { + t.Fatal(err) + } + return certPEM, key, cert + } + _, _, joined := issue("hall-b") + dir := t.TempDir() + p := filepath.Join(dir, "rack.pem") + if loadRenewed(p, pool, "hall-b", joined) != nil { + t.Error("nothing renewed yet") + } + time.Sleep(1100 * time.Millisecond) // a later NotAfter + certPEM, key, renewed := issue("hall-b") + if err := writeAtomic(p, append(certPEM, key...), 0o600); err != nil { + t.Fatal(err) + } + got := loadRenewed(p, pool, "hall-b", joined) + if got == nil || got.Leaf.SerialNumber.Cmp(renewed.SerialNumber) != 0 { + t.Fatalf("renewed certificate not used: %v", got) + } + if loadRenewed(p, pool, "hall-c", joined) != nil { + t.Error("another rack's certificate was used") + } + if loadRenewed(p, pool, "hall-b", renewed) != nil { + t.Error("an older or equal certificate replaced the current one") + } + other, _ := newCA("other CA") + op := x509.NewCertPool() + op.AddCert(other.Cert) + if loadRenewed(p, op, "hall-b", joined) != nil { + t.Error("a certificate from another CA was used") + } + if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 { + t.Errorf("mode %v", st.Mode()) + } +} diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 02f5125..70f9fa7 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -100,6 +100,13 @@ func main() { cmd, args := os.Args[1], os.Args[2:] switch cmd { case "rack": + if len(args) > 0 && args[0] == "renew" { + if err := rackRenew(args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + return + } if len(args) > 0 && args[0] == "join" { if err := rackJoin(args[1:]); err != nil { fmt.Fprintln(os.Stderr, "error:", err) diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 5f61bd8..0be7767 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -54,6 +54,14 @@ type RackConfig struct { KeyFile string `json:"key_file,omitempty"` // and key; with them the secret is not needed AutoUpdate *bool `json:"auto_update,omitempty"` // follow the region's version (default true) UpdateDir string `json:"update_dir,omitempty"` // default /var/lib/rollcall/update + // CertStateDir: where a renewed certificate goes (cert and key in one file, + // rack.pem). The files from rack join are root's; this directory is the + // service user's. Default /var/lib/rollcall/rack-cert. + CertStateDir string `json:"cert_state_dir,omitempty"` +} + +func (c RackConfig) renewedPath() string { + return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "rack.pem") } type rackd struct { @@ -137,6 +145,9 @@ func (r *rackd) clients() error { } pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) r.cert = &pair + if renewed := loadRenewed(r.cfg.renewedPath(), pool, r.cfg.Name, pair.Leaf); renewed != nil { + r.cert = renewed + } r.tlsc.RootCAs = pool r.tlsc.VerifyConnection = regionLeaf r.tlsc.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { @@ -186,11 +197,9 @@ func (r *rackd) renew() error { return err } pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) - if err := os.WriteFile(r.cfg.KeyFile, key, 0o640); err != nil { - return err - } - if err := os.WriteFile(r.cfg.CertFile, []byte(resp.Cert), 0o644); err != nil { - return err + // one file, renamed into place: a crash never leaves a key without its certificate + if err := writeAtomic(r.cfg.renewedPath(), append([]byte(resp.Cert), key...), 0o600); err != nil { + return fmt.Errorf("keep the renewed certificate (the region already switched to it; until this works the previous one is accepted for 7 days): %w; create the directory: install -d -m 0700 -o rollcall -g rollcall %s", err, filepath.Dir(r.cfg.renewedPath())) } r.certMu.Lock() r.cert = &pair @@ -199,6 +208,84 @@ func (r *rackd) renew() error { return nil } +// loadRenewed is the certificate renew() kept, when it belongs to this rack, +// comes from the CA and outlives the one from rack join. +func loadRenewed(p string, pool *x509.CertPool, name string, current *x509.Certificate) *tls.Certificate { + b, err := os.ReadFile(p) + if err != nil { + return nil + } + pair, err := tls.X509KeyPair(b, b) + if err != nil { + log.Printf("%s: %v (ignored)", p, err) + return nil + } + leaf, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil || leaf.Subject.CommonName != "rack:"+name { + return nil + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { + log.Printf("%s: %v (ignored)", p, err) + return nil + } + if current != nil && !leaf.NotAfter.After(current.NotAfter) { + return nil + } + pair.Leaf = leaf + return &pair +} + +func writeAtomic(p string, data []byte, mode os.FileMode) error { + f, err := os.CreateTemp(filepath.Dir(p), ".tmp-*") + if err != nil { + return err + } + defer os.Remove(f.Name()) + if _, err := f.Write(data); err != nil { + f.Close() + return err + } + if err := f.Chmod(mode); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Rename(f.Name(), p) +} + +// rackRenew renews the certificate now (rollcall rack renew), as the service user. +func rackRenew(args []string) error { + fs := newFlags("rack renew") + cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration") + _ = fs.Parse(args) + cfg, err := loadRackConfig(*cfgPath) + if err != nil { + return err + } + ru, err := url.Parse(cfg.Region) + if err != nil { + return err + } + r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}} + if err := r.clients(); err != nil { + return err + } + if r.cert == nil { + return errors.New("this rack controller has no certificate (rollcall rack join)") + } + if err := r.renew(); err != nil { + return err + } + fmt.Printf("renewed: %s, valid until %s; restart rollcall-rack to use it\n", cfg.renewedPath(), r.cert.Leaf.NotAfter.Format("2006-01-02")) + return nil +} + func newCSR(cn string) (keyPEM []byte, csrPEM string, err error) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index f30d9d7..bb8b492 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -164,6 +164,8 @@ CREATE TABLE IF NOT EXISTS rack_controllers ( registered timestamptz NOT NULL DEFAULT now() ); ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text; ALTER TABLE vlans ADD COLUMN IF NOT EXISTS dhcp_mode text NOT NULL DEFAULT ''; diff --git a/deploy/install-rack.sh b/deploy/install-rack.sh index fa30380..f20c56a 100755 --- a/deploy/install-rack.sh +++ b/deploy/install-rack.sh @@ -53,6 +53,7 @@ install -d -m 0755 /var/lib/rollcall-dnsmasq install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update +install -d -m 0700 -o rollcall -g rollcall /var/lib/rollcall/rack-cert # renewed certificates (the joined ones are root's) install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/ systemctl daemon-reload