From 2f708e8f69ca8670146180b222e309d3239e82ca Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 31 Oct 2026 15:37:44 +0100 Subject: [PATCH] Root rotation without re-joining: prepare, switch, retire The CA directory holds the active root and, during a rotation, the next or the previous one; everything that trusts the region gets the trust bundle (all of them). Rack controllers receive it in their state, signed by the active root; the root update unit installs it only when the signature verifies against the roots it already trusts, and binary signatures verify against any of them. After switch racks renew their client and server certificates from the new root by themselves and report what they run on; switch and retire wait for every rack (-force). /ca.pem stays one certificate, the root of that server's certificate, and the PXE command line names that root, so agents in older images keep working; /ca-bundle.pem is the bundle. Deployed machines get one file per root and a daily timer that refreshes them from their rack over HTTPS. Refs #12 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 + README.md | 32 +- cmd/rollcall/admincert.go | 2 +- cmd/rollcall/agenttls.go | 4 +- cmd/rollcall/ca.go | 56 +-- cmd/rollcall/carot.go | 673 +++++++++++++++++++++++++++++++ cmd/rollcall/carot_test.go | 178 ++++++++ cmd/rollcall/deploy.go | 19 +- cmd/rollcall/main.go | 3 +- cmd/rollcall/rackctl.go | 20 +- cmd/rollcall/rackd.go | 16 +- cmd/rollcall/server.go | 6 +- cmd/rollcall/stateless.go | 2 +- cmd/rollcall/store.go | 3 + deploy/rollcall-rack-update.path | 3 +- deploy/rollcall-rack-update.sh | 65 ++- 16 files changed, 1023 insertions(+), 71 deletions(-) create mode 100644 cmd/rollcall/carot.go create mode 100644 cmd/rollcall/carot_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d5def79..e4a8710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ publish a tag that has no section here. ## [Unreleased] +### Added +- Root rotation without re-joining (`rollcall ca rotate prepare | switch | retire`, status without + an argument). The trust bundle (all current roots, `/ca-bundle.pem`) reaches rack controllers + signed by the active root and is installed by their root unit only when that signature verifies; + racks renew client and server certificates from the new root by themselves; `switch` and + `retire` wait for every rack. Deployed machines get one file per root and a daily + `rollcall-ca-refresh` timer that fetches the bundle from their rack over HTTPS. + Upgrading: rerun `deploy/install-rack.sh` on rack hosts (the update unit installs bundles now). + +### Changed +- `rollcall ca rotate -yes` (replace the root, racks join again) is gone; the three steps replace it. + ## [0.3.0] - 2026-10-30 Third release: working with a site's existing network services (DNS provider, fixed leases on diff --git a/README.md b/README.md index 1d52de5..d6ae315 100644 --- a/README.md +++ b/README.md @@ -558,10 +558,34 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and short-lived certificates to nodes with certbot or acme.sh; everything that trusts the Rollcall CA trusts them, and nothing outside the permitted names can be issued. Roots created before this (path length 0) cannot delegate: - `rollcall ca rotate -yes` replaces the root, keeping the old one as a - backup; rack controllers then join again and clients trust the new - fingerprint. `deploy/tags/acme-cert.yaml` is a tag that fetches a node - certificate at first boot and renews it. + rotate the root (below). `deploy/tags/acme-cert.yaml` is a tag that fetches + a node certificate at first boot and renews it. +* **Root rotation without re-joining.** In three steps, each followed by a + restart of the region (`systemctl restart rollcall`): + + ``` + rollcall ca rotate prepare # a new root; the old one still signs + rollcall ca rotate # status: which rack has what + rollcall ca rotate switch # the new root signs; the old one stays trusted + rollcall ca rotate retire # the old root is no longer trusted + ``` + + Everything that trusts the region gets the *trust bundle* (all current + roots, `/ca-bundle.pem`). Rack controllers receive it signed by the active + root and their root unit installs it only if that signature verifies + against the roots they already trust; after `switch` they renew their + client and server certificates from the new root by themselves. `switch` + waits until every rack has the bundle, `retire` until every rack runs on + certificates from the new root (`-force` goes on; racks that missed it + join again). Machines deployed by this version get the roots one per file + and a daily `rollcall-ca-refresh` timer that fetches the bundle over HTTPS + from their rack, checked against the roots they trust, so they pick up a + new root during `prepare` and drop the old one after `retire`. Machines + deployed earlier need the bundle installed once by hand + (`curl --cacert https:///ca-bundle.pem`). Admin + certificates and intermediates signed by the old root work until `retire`; + issue them again after `switch`. `/ca.pem` stays a single certificate (the + root of that server's own certificate), which agents check by fingerprint. ## Power drivers diff --git a/cmd/rollcall/admincert.go b/cmd/rollcall/admincert.go index cc2aa3e..2ffa781 100644 --- a/cmd/rollcall/admincert.go +++ b/cmd/rollcall/admincert.go @@ -214,7 +214,7 @@ func (a *App) handleAdminCerts(w http.ResponseWriter, r *http.Request) { } a.dropAdminCertCache() log.Printf("admin certificate %q (serial %s, until %s) issued by %s", req.Name, cert.SerialNumber.Text(16), cert.NotAfter.Format("2006-01-02"), by) - writeJSON(w, 200, obj{"name": req.Name, "serial": cert.SerialNumber.Text(16), "expires": cert.NotAfter, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": req.Name, "serial": cert.SerialNumber.Text(16), "expires": cert.NotAfter, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } } diff --git a/cmd/rollcall/agenttls.go b/cmd/rollcall/agenttls.go index 0d9ab57..0859701 100644 --- a/cmd/rollcall/agenttls.go +++ b/cmd/rollcall/agenttls.go @@ -68,7 +68,7 @@ func (a *App) handleRackServerCert(w http.ResponseWriter, r *http.Request, rack return } log.Printf("rack controller %s: server certificate for %s until %s", rack, host, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // rackHost is the host part of a rack URL (an address or a name). @@ -157,7 +157,7 @@ func (a *App) agentTLSArgs(base string) []string { if t == "" { return nil } - fp := certFingerprint(a.ca.Cert) + fp := certFingerprint(a.serverCAFor(base)) // the root the rack serves as /ca.pem return []string{"rollcall.tls=" + t, "rollcall.ca=" + hex.EncodeToString(fp[:])} } diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index 7465cc0..78751cc 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -55,7 +55,11 @@ const ( type CA struct { Cert *x509.Certificate Key *ecdsa.PrivateKey - PEM []byte + PEM []byte // the active root + // all trusted roots, active first: during a rotation also the next or + // previous one (carot.go); what clients and machines are given to trust + Trust []*x509.Certificate + TrustPEM []byte } func randomSerial() (*big.Int, error) { @@ -121,7 +125,11 @@ func loadOrCreateCA(dir, cn string) (*CA, error) { if err != nil { return nil, fmt.Errorf("%s: %w", kp, err) } - return &CA{Cert: cert, Key: key, PEM: cb}, nil + ca := &CA{Cert: cert, Key: key, PEM: cb} + if err := ca.loadTrust(dir); err != nil { + return nil, err + } + return ca, nil } ca, err := newCA(cn) if err != nil { @@ -138,6 +146,7 @@ func loadOrCreateCA(dir, cn string) (*CA, error) { return nil, err } log.Printf("created certificate authority %q in %s (fingerprint %s)", cn, dir, ca.Fingerprint()) + ca.Trust, ca.TrustPEM = []*x509.Certificate{ca.Cert}, ca.PEM return ca, nil } @@ -609,7 +618,7 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) { a.reloadRacks(r.Context()) a.hub.bump() log.Printf("rack controller %s enrolled from %s, certificate until %s", name, r.RemoteAddr, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // handleRackRenew issues a new certificate to a rack controller that still has a valid one. @@ -633,7 +642,7 @@ func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack strin return } log.Printf("rack controller %s renewed its certificate, until %s", rack, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // ---------------------------------------------------------------- CLI: rollcall ca (on the region host) @@ -654,6 +663,13 @@ func caCmd(args []string) error { } fmt.Printf("%s\n subject %s\n valid until %s\n fingerprint %s\n", filepath.Join(cfg.CADir, "ca.pem"), c.Subject.CommonName, c.NotAfter.Format("2006-01-02"), fpString(certFingerprint(c))) + for _, f := range []string{"ca-next.pem", "ca-prev.pem"} { + if b, err := os.ReadFile(filepath.Join(cfg.CADir, f)); err == nil { + if o, err := parseCertPEM(b); err == nil { + fmt.Printf("%s (also trusted, rotation)\n fingerprint %s\n", f, fpString(certFingerprint(o))) + } + } + } if rb, err := os.ReadFile(filepath.Join(cfg.CADir, "region.pem")); err == nil { if rc, err := parseCertPEM(rb); err == nil { names := append([]string{}, rc.DNSNames...) @@ -777,38 +793,6 @@ func caIntermediateCmd(cfg *Config, name string, args []string) error { return nil } -// rollcall ca rotate -yes: a new root (the old one is kept as a backup). Racks -// must join again and clients trust the new fingerprint. -func caRotateCmd(cfg *Config, args []string) error { - fs := newFlags("ca rotate") - yes := fs.Bool("yes", false, "really replace the root") - _ = fs.Parse(args) - old, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - return err - } - if !*yes { - return fmt.Errorf("this replaces the root %s (delegates: %v); rack controllers then join again and clients trust the new one. Run with -yes", old.Fingerprint(), old.canDelegate()) - } - bak := cfg.CADir + ".bak-" + time.Now().Format("20060102-150405") - if err := os.Rename(cfg.CADir, bak); err != nil { - return err - } - ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - _ = os.Rename(bak, cfg.CADir) - return err - } - if st, err := os.Stat(bak); err == nil { - _ = os.Chmod(cfg.CADir, st.Mode().Perm()) - if own, ok := fileOwner(bak); ok { - _ = chownTree(cfg.CADir, own[0], own[1]) - } - } - fmt.Printf("new root %s\nold root kept in %s\nnext: restart rollcall; on each rack controller `rollcall rack join` with a new join code; clients trust the new fingerprint\n", ca.Fingerprint(), bak) - return nil -} - // fileOwner returns uid, gid of path (unix). func fileOwner(path string) ([2]int, bool) { st, err := os.Stat(path) diff --git a/cmd/rollcall/carot.go b/cmd/rollcall/carot.go new file mode 100644 index 0000000..9ffe1a6 --- /dev/null +++ b/cmd/rollcall/carot.go @@ -0,0 +1,673 @@ +package main + +// Root rotation without re-joining. The CA directory holds the active root +// (ca.pem, ca-key.pem) and, during a rotation, the next one (ca-next.pem, +// ca-next-key.pem) or the previous one (ca-prev.pem). Everything that trusts +// the region gets the trust bundle (all of them, active first): +// +// prepare a new root is made; the old one still signs. Rack controllers +// install the bundle (signed by the active root, installed by their +// root unit), machines deployed from now on trust both. +// switch the new root signs: the region's certificate, rack and admin +// certificates from now on. Racks renew their client and server +// certificates by themselves; the old root stays trusted. +// retire the old root leaves the bundle once every rack runs on the new one. +// +// switch and retire refuse while a rack is not ready (-force overrides). +// /ca.pem stays one certificate (the root of the server's own certificate: +// agents in older images read one); /ca-bundle.pem is the bundle. + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// loadTrust adds the next and previous roots of a rotation to ca.Trust. +func (ca *CA) loadTrust(dir string) error { + ca.Trust = []*x509.Certificate{ca.Cert} + ca.TrustPEM = append([]byte{}, ca.PEM...) + for _, f := range []string{"ca-next.pem", "ca-prev.pem"} { + b, err := os.ReadFile(filepath.Join(dir, f)) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + c, err := parseCertPEM(b) + if err != nil || !c.IsCA { + return fmt.Errorf("%s: not a CA certificate", f) + } + ca.Trust = append(ca.Trust, c) + ca.TrustPEM = append(ca.TrustPEM, pemEncode("CERTIFICATE", c.Raw)...) + } + return nil +} + +func fpHex(c *x509.Certificate) string { + sum := certFingerprint(c) + return hex.EncodeToString(sum[:]) +} + +func (ca *CA) pool() *x509.CertPool { + p := x509.NewCertPool() + for _, c := range ca.Trust { + p.AddCert(c) + } + return p +} + +// signBytes is the active root's signature over data's SHA-256 (base64 DER), +// as for rack binaries: openssl dgst -sha256 -verify checks it. +func (ca *CA) signBytes(data []byte) string { + sum := sha256.Sum256(data) + der, err := ecdsa.SignASN1(rand.Reader, ca.Key, sum[:]) + if err != nil { + return "" + } + return base64.StdEncoding.EncodeToString(der) +} + +// rootOf is the root among trust that signed c, nil when none did. +func rootOf(c *x509.Certificate, trust []*x509.Certificate) *x509.Certificate { + if c == nil { + return nil + } + for _, r := range trust { + if bytes.Equal(c.RawIssuer, r.RawSubject) && c.CheckSignatureFrom(r) == nil { + return r + } + } + return nil +} + +// verifyBundle: a bundle the region sent, signed by one of the roots trusted +// now, holding CA certificates only. +func verifyBundle(bundle []byte, sig string, trust []*x509.Certificate) ([]*x509.Certificate, error) { + der, err := base64.StdEncoding.DecodeString(sig) + if err != nil { + return nil, fmt.Errorf("bundle signature: %w", err) + } + sum := sha256.Sum256(bundle) + ok := false + for _, r := range trust { + if pub, isEC := r.PublicKey.(*ecdsa.PublicKey); isEC && ecdsa.VerifyASN1(pub, sum[:], der) { + ok = true + } + } + if !ok { + return nil, errors.New("the bundle is not signed by a root trusted here") + } + certs, err := parsePEMCerts(bundle) + if err != nil { + return nil, err + } + for _, c := range certs { + if !c.IsCA { + return nil, fmt.Errorf("bundle: %q is not a CA certificate", c.Subject.CommonName) + } + } + return certs, nil +} + +func parsePEMCerts(b []byte) ([]*x509.Certificate, error) { + var out []*x509.Certificate + for { + var blk *pem.Block + blk, b = pem.Decode(b) + if blk == nil { + break + } + if blk.Type != "CERTIFICATE" { + return nil, fmt.Errorf("unexpected PEM block %s", blk.Type) + } + c, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + return nil, err + } + out = append(out, c) + } + if len(out) == 0 { + return nil, errors.New("no certificate") + } + return out, nil +} + +// ---------------------------------------------------------------- store + +// SetRackTrust records what a rack runs on: the hash of its trust bundle and +// the roots of its client and server certificates. +func (s *Store) SetRackTrust(ctx context.Context, name, bundleSHA, clientCA, serverCA string) error { + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET ca_bundle_sha256=nullif($2,''), client_ca=nullif($3,''), + server_ca=coalesce(nullif($4,''), server_ca) WHERE name=$1`, name, bundleSHA, clientCA, serverCA) + return err +} + +type rackTrust struct { + Name string `json:"name"` + Bundle string `json:"ca_bundle_sha256"` + ClientCA string `json:"client_ca"` + ServerCA string `json:"server_ca"` + LastSeen *time.Time `json:"last_seen"` +} + +func (s *Store) RackTrust(ctx context.Context) ([]rackTrust, error) { + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(ca_bundle_sha256,''), coalesce(client_ca,''), coalesce(server_ca,''), last_seen + FROM rack_controllers WHERE kind='remote' ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []rackTrust{} + for rows.Next() { + var t rackTrust + if err := rows.Scan(&t.Name, &t.Bundle, &t.ClientCA, &t.ServerCA, &t.LastSeen); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +// ---------------------------------------------------------------- region HTTP + +func (a *App) handleCABundle(w http.ResponseWriter, r *http.Request) { + if a.ca == nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-pem-file") + _, _ = w.Write(a.ca.TrustPEM) +} + +type rotationStatus struct { + Phase string `json:"phase"` // "" | prepared | switched + Active string `json:"active"` + Next string `json:"next,omitempty"` + Prev string `json:"prev,omitempty"` + Bundle string `json:"bundle_sha256"` + Racks []rackTrust `json:"racks"` + // what blocks the next step + NotReady []string `json:"not_ready"` +} + +func (a *App) rotation(ctx context.Context) (rotationStatus, error) { + st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}} + sum := sha256.Sum256(a.ca.TrustPEM) + st.Bundle = hex.EncodeToString(sum[:]) + if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-next.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + st.Phase, st.Next = "prepared", fpHex(c) + } + } + if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + st.Phase, st.Prev = "switched", fpHex(c) + } + } + racks, err := a.store.RackTrust(ctx) + if err != nil { + return st, err + } + st.Racks = racks + for _, r := range racks { + switch st.Phase { + case "prepared": // switch needs the new root at every rack + if r.Bundle != st.Bundle { + st.NotReady = append(st.NotReady, r.Name+": has not installed the bundle with the new root") + } + case "switched": // retire needs every rack on certificates from the new root + if r.ClientCA != st.Active || r.ServerCA != "" && r.ServerCA != st.Active { + st.NotReady = append(st.NotReady, r.Name+": still has a certificate from the old root") + } + } + } + return st, nil +} + +func (a *App) handleCARotation(w http.ResponseWriter, r *http.Request) { + if a.ca == nil { + fail(w, 409, errors.New("no CA")) + return + } + st, err := a.rotation(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, st) +} + +// noteRackTrust records, at registration, the root of the client certificate +// the rack connected with and what it says about its bundle and server certificate. +func (a *App) noteRackTrust(r *http.Request, rack string, rc RackController) { + if a.ca == nil || r.TLS == nil { + return + } + var client string + for _, ch := range r.TLS.VerifiedChains { + if len(ch) == 2 { + client = fpHex(ch[1]) + } + } + _ = a.store.SetRackTrust(r.Context(), rack, rc.CABundleSHA256, client, rc.ServerCA) +} + +// serverCAFor is the root an agent behind this rack must accept: that of the +// rack's server certificate (it serves that one as /ca.pem). +func (a *App) serverCAFor(base string) *x509.Certificate { + if base != a.cfg.ServerURL { + for _, rc := range a.racksSnapshot() { + if strings.TrimRight(rc.URL, "/") == base && rc.ServerCA != "" { + for _, c := range a.ca.Trust { + if fpHex(c) == rc.ServerCA { + return c + } + } + } + } + } + return a.ca.Cert +} + +var trustSig struct { + sync.Mutex + bundle, sig string +} + +// trustState is signed once per bundle: the rack state stays the same between polls. +func (a *App) trustState() *rackTrustState { + trustSig.Lock() + defer trustSig.Unlock() + if trustSig.bundle != string(a.ca.TrustPEM) { + trustSig.bundle, trustSig.sig = string(a.ca.TrustPEM), a.ca.signBytes(a.ca.TrustPEM) + } + return &rackTrustState{Bundle: trustSig.bundle, Sig: trustSig.sig, Active: fpHex(a.ca.Cert)} +} + +// caRefreshFiles keep a deployed machine's roots current: the bundle comes +// over HTTPS from the rack, checked against the roots the machine trusts now, +// so a rotation reaches it during prepare (before the new root signs) and the +// old root leaves it after retire. +func caRefreshFiles(tlsBase string) []seedFile { + script := `#!/bin/sh +# rollcall: keep the installation's roots current (written at deploy time) +set -eu +URL=` + shQuote(tlsBase+"/ca-bundle.pem") + ` +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +curl -fsS --max-time 30 -o "$T/bundle" "$URL" || exit 0 # unreachable: the timer tries again +awk -v d="$T" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n)} /-----END CERTIFICATE-----/{close(d "/c" n)}' "$T/bundle" +[ -f "$T/c1" ] || exit 0 +changed= +for dir in /usr/local/share/ca-certificates /etc/pki/ca-trust/source/anchors; do + [ -d "$dir" ] || continue + i=0; mkdir -p "$T/want"; rm -f "$T"/want/* + for c in "$T"/c[0-9]*; do + i=$((i+1)); n=rollcall-ca.crt; [ $i -gt 1 ] && n=rollcall-ca-$i.crt + cp "$c" "$T/want/$n" + done + if [ "$(cat "$dir"/rollcall-ca*.crt 2>/dev/null)" != "$(cat "$T"/want/*)" ]; then + rm -f "$dir"/rollcall-ca*.crt + for f in "$T"/want/*; do install -m 0644 "$f" "$dir/$(basename "$f")"; done + changed=1 + fi +done +[ -n "$changed" ] || exit 0 +if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi +echo "rollcall: trusted roots updated from $URL" +` + return []seedFile{ + {Path: "usr/local/sbin/rollcall-ca-refresh", Mode: "0755", Content: script}, + {Path: "etc/systemd/system/rollcall-ca-refresh.service", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/rollcall-ca-refresh\n"}, + {Path: "etc/systemd/system/rollcall-ca-refresh.timer", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots daily\n\n[Timer]\nOnBootSec=10min\nOnUnitActiveSec=1d\nRandomizedDelaySec=1h\n\n[Install]\nWantedBy=timers.target\n"}, + } +} + +// ---------------------------------------------------------------- CLI + +// rollcall ca rotate [status | prepare | switch | retire] [-force] +func caRotateCmd(cfg *Config, args []string) error { + sub := "status" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + sub, args = args[0], args[1:] + } + fs := newFlags("ca rotate " + sub) + force := fs.Bool("force", false, "go on although rack controllers are not ready (they then join again)") + _ = fs.Parse(args) + dir := cfg.CADir + have := func(f string) bool { _, err := os.Stat(filepath.Join(dir, f)); return err == nil } + status := func() (*rotationStatus, error) { + c, err := newClient() + if err != nil { + return nil, err + } + var st rotationStatus + if err := c.do("GET", "/api/v1/ca/rotation", nil, &st); err != nil { + return nil, err + } + return &st, nil + } + ready := func() error { + st, err := status() + if err != nil { + if *force { + fmt.Fprintf(os.Stderr, "cannot ask the region (%v); -force: going on\n", err) + return nil + } + return fmt.Errorf("cannot ask the region whether rack controllers are ready: %w (-force to go on)", err) + } + if len(st.NotReady) > 0 && !*force { + return fmt.Errorf("not ready:\n %s\nwait for them (they pick it up within minutes when online) or -force (they then join again)", strings.Join(st.NotReady, "\n ")) + } + return nil + } + restart := "next: restart rollcall (systemctl restart rollcall); `rollcall ca rotate` shows the progress" + switch sub { + case "status": + st, err := status() + if err != nil { + return err + } + phase := map[string]string{"": "no rotation", "prepared": "prepared (next: switch)", "switched": "switched (next: retire)"}[st.Phase] + fmt.Printf("phase %s\nactive %s\n", phase, fpString(mustFP(st.Active))) + if st.Next != "" { + fmt.Printf("next %s\n", fpString(mustFP(st.Next))) + } + if st.Prev != "" { + fmt.Printf("prev %s (still trusted)\n", fpString(mustFP(st.Prev))) + } + short := func(fp string) string { + switch { + case fp == "": + return "-" + case fp == st.Active: + return "active" + case fp == st.Next: + return "next" + case fp == st.Prev: + return "prev" + } + return fp[:12] + } + fmt.Printf("\n%-16s %-8s %-8s %-8s %s\n", "RACK", "BUNDLE", "CLIENT", "SERVER", "LAST SEEN") + for _, r := range st.Racks { + b := "old" + if r.Bundle == st.Bundle { + b = "current" + } + seen := "-" + if r.LastSeen != nil { + seen = r.LastSeen.Local().Format("2006-01-02 15:04") + } + fmt.Printf("%-16s %-8s %-8s %-8s %s\n", r.Name, b, short(r.ClientCA), short(r.ServerCA), seen) + } + for _, n := range st.NotReady { + fmt.Println("waiting:", n) + } + return nil + case "prepare": + if have("ca-next.pem") || have("ca-prev.pem") { + return errors.New("a rotation is under way (rollcall ca rotate status)") + } + next, err := newCA("rollcall CA " + time.Now().Format("2006-01-02")) + if err != nil { + return err + } + if err := writeCA(dir, "ca-next", next); err != nil { + return err + } + fmt.Printf("next root %s\nthe active root still signs; rack controllers and newly deployed machines get both\n%s; then `rollcall ca rotate switch`\n", next.Fingerprint(), restart) + return nil + case "switch": + if !have("ca-next.pem") { + return errors.New("no prepared root (rollcall ca rotate prepare)") + } + if err := ready(); err != nil { + return err + } + bak := filepath.Join(dir, "rotated-"+time.Now().Format("20060102-150405")) + if err := os.Mkdir(bak, 0o700); err != nil { + return err + } + steps := [][2]string{{"ca.pem", "ca-prev.pem"}, {"ca-key.pem", filepath.Join(filepath.Base(bak), "ca-key.pem")}, + {"ca-next-key.pem", "ca-key.pem"}, {"ca-next.pem", "ca.pem"}} + for _, s := range steps { + if err := copyKeepOwner(filepath.Join(dir, s[0]), filepath.Join(dir, s[1])); err != nil { + return fmt.Errorf("%s -> %s: %w (the CA directory may be half switched; the old files are in %s)", s[0], s[1], err, bak) + } + } + for _, f := range []string{"ca-next.pem", "ca-next-key.pem"} { + _ = os.Remove(filepath.Join(dir, f)) + } + fmt.Printf("the new root signs now; the old one stays trusted (its key: %s)\nrack controllers renew their certificates by themselves; admin certificates and intermediates from the old root work until retire: issue them again\n%s; then `rollcall ca rotate retire`\n", bak, restart) + return nil + case "retire": + if !have("ca-prev.pem") { + return errors.New("no previous root to retire") + } + if err := ready(); err != nil { + return err + } + bak := filepath.Join(dir, "retired-"+time.Now().Format("20060102-150405")+".pem") + if err := os.Rename(filepath.Join(dir, "ca-prev.pem"), bak); err != nil { + return err + } + fmt.Printf("the old root is no longer trusted (kept in %s)\n%s\n", bak, restart) + return nil + } + return fmt.Errorf("ca rotate %s: unknown (status, prepare, switch, retire)", sub) +} + +func mustFP(h string) [32]byte { + var out [32]byte + b, _ := hex.DecodeString(h) + copy(out[:], b) + return out +} + +// writeCA writes a root as .pem / -key.pem, owned like ca.pem. +func writeCA(dir, name string, ca *CA) error { + kder, err := x509.MarshalECPrivateKey(ca.Key) + if err != nil { + return err + } + kp, cp := filepath.Join(dir, name+"-key.pem"), filepath.Join(dir, name+".pem") + if err := os.WriteFile(kp, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil { + return err + } + if err := os.WriteFile(cp, ca.PEM, 0o644); err != nil { + return err + } + if own, ok := fileOwner(filepath.Join(dir, "ca.pem")); ok { + _ = os.Chown(kp, own[0], own[1]) + _ = os.Chown(cp, own[0], own[1]) + } + return nil +} + +// copyKeepOwner copies src over dst (atomically) with src's mode and owner. +func copyKeepOwner(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + st, err := os.Stat(src) + if err != nil { + return err + } + if err := writeAtomic(dst, b, st.Mode().Perm()); err != nil { + return err + } + if own, ok := fileOwner(src); ok { + _ = os.Chown(dst, own[0], own[1]) + } + return nil +} + +// shQuote is s as one single-quoted shell word. +func shQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// ---------------------------------------------------------------- rack controller + +// rackTrust is what the rack trusts now (its ca_file, written at join and by +// the root update unit). +func (r *rackd) rackTrust() ([]byte, []*x509.Certificate) { + b, err := os.ReadFile(r.cfg.CAFile) + if err != nil { + return nil, nil + } + certs, _ := parsePEMCerts(b) + return b, certs +} + +// trustReport: the hash of the rack's roots and the root of its server certificate. +func (r *rackd) trustReport() (bundleSHA, serverCA string) { + b, trust := r.rackTrust() + if b == nil { + return "", "" + } + sum := sha256.Sum256(b) + r.srvMu.Lock() + c := r.srvCert + r.srvMu.Unlock() + if c != nil { + if root := rootOf(c.Leaf, trust); root != nil { + serverCA = fpHex(root) + } + } + return hex.EncodeToString(sum[:]), serverCA +} + +// serveCA answers /ca.pem with the root of the server certificate (agents +// pin it by fingerprint and read one certificate) and /ca-bundle.pem with all. +func (r *rackd) serveCA(bundle bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + b, trust := r.rackTrust() + if b == nil { + http.Error(w, "no CA", 404) + return + } + w.Header().Set("Content-Type", "application/x-pem-file") + if bundle || len(trust) < 2 { + _, _ = w.Write(b) + return + } + r.srvMu.Lock() + c := r.srvCert + r.srvMu.Unlock() + root := trust[0] + if c != nil { + if o := rootOf(c.Leaf, trust); o != nil { + root = o + } + } + _, _ = w.Write(pemEncode("CERTIFICATE", root.Raw)) + } +} + +var rackTrustMu sync.Mutex +var rackTrustTried = map[string]time.Time{} + +// maybeTrust follows the region's roots: a new bundle is staged for the root +// unit, and certificates from a root that no longer signs are renewed. +func (r *rackd) maybeTrust(t *rackTrustState) { + if t == nil || r.cert == nil { + return + } + rackTrustMu.Lock() + defer rackTrustMu.Unlock() + cur, trust := r.rackTrust() + if t.Bundle != "" && t.Bundle != string(cur) { + sum := sha256.Sum256([]byte(t.Bundle)) + key := "bundle " + hex.EncodeToString(sum[:]) + if time.Since(rackTrustTried[key]) > time.Hour { + rackTrustTried[key] = time.Now() + if err := r.stageBundle(t, trust); err != nil { + log.Printf("trust bundle: %v", err) + } else { + log.Printf("new trust bundle staged in %s; rollcall-rack-update installs it", r.updateDir()) + } + } + } + var active *x509.Certificate + for _, c := range trust { + if fpHex(c) == t.Active { + active = c + } + } + if active == nil { + return // the signing root is not trusted here yet: the bundle comes first + } + renewed := false + r.certMu.Lock() + leaf := r.cert.Leaf + r.certMu.Unlock() + if rootOf(leaf, trust) != active && time.Since(rackTrustTried["client"]) > 10*time.Minute { + rackTrustTried["client"] = time.Now() + if err := r.renew(); err != nil { + log.Printf("certificate from the new root: %v", err) + } else { + renewed = true + } + } + r.srvMu.Lock() + srv := r.srvCert + r.srvMu.Unlock() + if srv != nil && rootOf(srv.Leaf, trust) != active && time.Since(rackTrustTried["server"]) > 10*time.Minute { + rackTrustTried["server"] = time.Now() + if c, err := r.newServerCert(); err != nil { + log.Printf("server certificate from the new root: %v", err) + } else { + r.srvMu.Lock() + r.srvCert = c + r.srvMu.Unlock() + renewed = true + } + } + if renewed { + for _, hc := range []*http.Client{r.hc, r.fast, r.big} { // new connections present the new certificate + if tr, ok := hc.Transport.(*http.Transport); ok { + tr.CloseIdleConnections() + } + } + if err := r.register(); err != nil { + log.Printf("register after renewal: %v", err) + } + } +} + +// stageBundle leaves a verified bundle and its signature for the root unit, +// which checks the signature again against its own copy of the roots. +func (r *rackd) stageBundle(t *rackTrustState, trust []*x509.Certificate) error { + if _, err := verifyBundle([]byte(t.Bundle), t.Sig, trust); err != nil { + return err + } + sig, _ := base64.StdEncoding.DecodeString(t.Sig) + dir := r.updateDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := writeAtomic(filepath.Join(dir, "ca-bundle.pem"), []byte(t.Bundle), 0o644); err != nil { + return err + } + if err := writeAtomic(filepath.Join(dir, "ca-bundle.sig"), sig, 0o644); err != nil { + return err + } + return writeAtomic(filepath.Join(dir, "ca-ready"), []byte("1\n"), 0o644) +} diff --git a/cmd/rollcall/carot_test.go b/cmd/rollcall/carot_test.go new file mode 100644 index 0000000..b54fe9b --- /dev/null +++ b/cmd/rollcall/carot_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func rotTestCA(t *testing.T) (string, *Config) { + t.Helper() + dir := filepath.Join(t.TempDir(), "ca") + if _, err := loadOrCreateCA(dir, "rollcall CA"); err != nil { + t.Fatal(err) + } + return dir, &Config{CADir: dir} +} + +func TestCARotationPhases(t *testing.T) { + dir, cfg := rotTestCA(t) + old, _ := loadOrCreateCA(dir, "rollcall CA") + if len(old.Trust) != 1 || !bytes.Equal(old.TrustPEM, old.PEM) { + t.Fatalf("no rotation: trust %d", len(old.Trust)) + } + leafOld, _, err := old.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + if err != nil { + t.Fatal(err) + } + // prepare: the old root still signs, both are trusted + if err := caRotateCmd(cfg, []string{"prepare"}); err != nil { + t.Fatal(err) + } + if err := caRotateCmd(cfg, []string{"prepare"}); err == nil { + t.Fatal("a second prepare during a rotation") + } + p, _ := loadOrCreateCA(dir, "rollcall CA") + if fpHex(p.Cert) != fpHex(old.Cert) || len(p.Trust) != 2 { + t.Fatalf("prepared: active %s trust %d", fpHex(p.Cert)[:8], len(p.Trust)) + } + next := p.Trust[1] + // switch (no region to ask: -force) + if err := caRotateCmd(cfg, []string{"switch", "-force"}); err != nil { + t.Fatal(err) + } + s, err := loadOrCreateCA(dir, "rollcall CA") + if err != nil { + t.Fatal(err) + } + if fpHex(s.Cert) != fpHex(next) || len(s.Trust) != 2 || fpHex(s.Trust[1]) != fpHex(old.Cert) { + t.Fatalf("switched: active should be the prepared root, the old one trusted") + } + if !s.Key.PublicKey.Equal(s.Cert.PublicKey) { + t.Fatal("the active key is not the active root's") + } + // a certificate from the old root still verifies, a new one comes from the new root + lc, _ := parseCertPEM(leafOld) + if _, err := lc.Verify(x509.VerifyOptions{Roots: s.pool(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { + t.Fatalf("old certificate after switch: %v", err) + } + nb, _, _ := s.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + nc, _ := parseCertPEM(nb) + if rootOf(nc, s.Trust) != s.Cert || rootOf(lc, s.Trust) == s.Cert { + t.Fatal("rootOf") + } + // the region's certificate is issued again by the new root + if _, err := regionCert(old, dir, []string{"region.example"}); err != nil { + t.Fatal(err) + } + pair, err := regionCert(s, dir, []string{"region.example"}) + if err != nil { + t.Fatal(err) + } + rc, _ := x509.ParseCertificate(pair.Certificate[0]) + if rc.CheckSignatureFrom(s.Cert) != nil { + t.Fatal("region certificate not from the new root after switch") + } + if err := caRotateCmd(cfg, []string{"switch", "-force"}); err == nil { + t.Fatal("switch without a prepared root") + } + // retire: only the new root + if err := caRotateCmd(cfg, []string{"retire", "-force"}); err != nil { + t.Fatal(err) + } + r, _ := loadOrCreateCA(dir, "rollcall CA") + if len(r.Trust) != 1 || fpHex(r.Cert) != fpHex(next) { + t.Fatal("retired: trust should be the new root only") + } + if _, err := lc.Verify(x509.VerifyOptions{Roots: r.pool(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err == nil { + t.Fatal("a certificate from the retired root still verifies") + } +} + +func TestTrustBundleSignature(t *testing.T) { + dir, cfg := rotTestCA(t) + if err := caRotateCmd(cfg, []string{"prepare"}); err != nil { + t.Fatal(err) + } + ca, _ := loadOrCreateCA(dir, "rollcall CA") + sig := ca.signBytes(ca.TrustPEM) + got, err := verifyBundle(ca.TrustPEM, sig, []*x509.Certificate{ca.Cert}) + if err != nil || len(got) != 2 { + t.Fatalf("valid bundle: %v (%d)", err, len(got)) + } + stranger, _ := newCA("someone") + if _, err := verifyBundle(ca.TrustPEM, sig, []*x509.Certificate{stranger.Cert}); err == nil { + t.Fatal("accepted a bundle signed by a root not trusted here") + } + if _, err := verifyBundle(append(append([]byte{}, ca.TrustPEM...), stranger.PEM...), sig, ca.Trust); err == nil { + t.Fatal("accepted a bundle with a root added after signing") + } + leaf, _, _ := ca.issue("x", []string{"x.example"}, x509.ExtKeyUsageServerAuth) + withLeaf := append(append([]byte{}, ca.TrustPEM...), leaf...) + if _, err := verifyBundle(withLeaf, ca.signBytes(withLeaf), ca.Trust); err == nil || !strings.Contains(err.Error(), "not a CA") { + t.Fatalf("a leaf in the bundle: %v", err) + } +} + +func TestRackStagesBundle(t *testing.T) { + dir, cfg := rotTestCA(t) + ca, _ := loadOrCreateCA(dir, "rollcall CA") + rd := t.TempDir() + caFile := filepath.Join(rd, "ca.pem") + _ = os.WriteFile(caFile, ca.PEM, 0o644) + certPEM, keyPEM, _ := ca.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + pair, _ := tls.X509KeyPair(certPEM, keyPEM) + pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) + r := &rackd{cfg: RackConfig{Name: "r1", CAFile: caFile, UpdateDir: filepath.Join(rd, "update")}, cert: &pair} + _ = caRotateCmd(cfg, []string{"prepare"}) + ca, _ = loadOrCreateCA(dir, "rollcall CA") + app := &App{ca: ca} + r.maybeTrust(app.trustState()) + b, err := os.ReadFile(filepath.Join(rd, "update", "ca-bundle.pem")) + if err != nil || !bytes.Equal(b, ca.TrustPEM) { + t.Fatalf("bundle not staged: %v", err) + } + if _, err := os.Stat(filepath.Join(rd, "update", "ca-ready")); err != nil { + t.Fatal("no ca-ready marker") + } + // the signature file is DER, for openssl dgst -verify + sig, _ := os.ReadFile(filepath.Join(rd, "update", "ca-bundle.sig")) + if len(sig) == 0 || sig[0] != 0x30 { + t.Fatal("signature not DER") + } + // a forged bundle is not staged + _ = os.RemoveAll(filepath.Join(rd, "update")) + stranger, _ := newCA("someone") + forged := append(append([]byte{}, ca.TrustPEM...), stranger.PEM...) + rackTrustTried = map[string]time.Time{} + r.maybeTrust(&rackTrustState{Bundle: string(forged), Sig: stranger.signBytes(forged), Active: fpHex(ca.Cert)}) + if _, err := os.Stat(filepath.Join(rd, "update", "ca-ready")); err == nil { + t.Fatal("staged a bundle signed by a stranger") + } + // the rack serves one certificate as /ca.pem: agents in older images read one + _ = os.WriteFile(caFile, ca.TrustPEM, 0o644) + if _, trust := r.rackTrust(); len(trust) != 2 { + t.Fatal("rack trust") + } +} + +func TestCARefreshScript(t *testing.T) { + fs := caRefreshFiles("https://10.213.78.1") + if len(fs) != 3 || !strings.Contains(fs[0].Content, "URL='https://10.213.78.1/ca-bundle.pem'") { + t.Fatalf("%+v", fs[0]) + } + f := filepath.Join(t.TempDir(), "s") + _ = os.WriteFile(f, []byte(fs[0].Content), 0o755) + if out, err := exec.Command("sh", "-n", f).CombinedOutput(); err != nil { + t.Fatalf("sh -n: %v %s", err, out) + } + if shQuote("a'b") != `'a'\''b'` { + t.Fatal("shQuote") + } +} diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index 3f39654..28fc7b2 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -311,7 +311,8 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { // cloud-init's ca_certs runs update-ca-certificates even on RHEL-family systems // and fails there, so it is not used. frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: "#cloud-config\n# rollcall: trust the installation's certificate authority\nbootcmd:\n" + - " - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n"}) + " - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n" + + " - [sh, -c, 'if [ -f /etc/systemd/system/rollcall-ca-refresh.timer ]; then systemctl enable --now rollcall-ca-refresh.timer || true; fi']\n"}) } if merr == nil { if tf, err := a.tagFragments(context.Background(), m); err == nil { @@ -327,9 +328,19 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { {Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"}, } if a.ca != nil { // both layouts; the bootcmd above picks the one the distribution uses - files = append(files, - seedFile{Path: "usr/local/share/ca-certificates/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)}, - seedFile{Path: "etc/pki/ca-trust/source/anchors/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)}) + for i, c := range a.ca.Trust { // one root per file (update-ca-certificates reads one) + name := "rollcall-ca.crt" + if i > 0 { + name = fmt.Sprintf("rollcall-ca-%d.crt", i+1) + } + pemC := string(pemEncode("CERTIFICATE", c.Raw)) + files = append(files, + seedFile{Path: "usr/local/share/ca-certificates/" + name, Mode: "0644", Content: pemC}, + seedFile{Path: "etc/pki/ca-trust/source/anchors/" + name, Mode: "0644", Content: pemC}) + } + if t := a.agentTLS(base); t != "" { // a root rotation reaches the machine later on + files = append(files, caRefreshFiles(t)...) + } } if sp.NetworkConfig != "" { files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig}) diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index d5effe6..63c92f9 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -32,7 +32,8 @@ func usage() { rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) - rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] | rotate -yes delegate to an ACME CA / replace the root + rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA + rollcall ca rotate [status | prepare | switch | retire] [-force] replace the root without re-joining racks rollcall version rollcall keygen rollcall list diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index af8e8e6..997d428 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -52,6 +52,9 @@ type RackController struct { Registered time.Time `json:"registered"` Online bool `json:"online"` Machines int `json:"machines"` + // trust (carot.go): what the rack reports at registration + CABundleSHA256 string `json:"ca_bundle_sha256,omitempty"` // SHA-256 of the roots it trusts + ServerCA string `json:"server_ca,omitempty"` // root of its server certificate (agents) } // rackState is everything a rack controller writes to disk. @@ -64,6 +67,15 @@ type rackState struct { Hosts string `json:"hosts"` BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run + Trust *rackTrustState `json:"trust,omitempty"` // the roots to trust (carot.go) +} + +// rackTrustState: the trust bundle signed by the active root, and which root +// signs now (a rack renews certificates from another one). +type rackTrustState struct { + Bundle string `json:"bundle"` + Sig string `json:"sig"` + Active string `json:"active"` // SHA-256 of the active root } type rackJob struct { @@ -91,7 +103,7 @@ type rackResult struct { // ---------------------------------------------------------------- store func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { - rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered, coalesce(server_ca,'') FROM rack_controllers ORDER BY name`) if err != nil { return nil, err } @@ -100,7 +112,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { for rows.Next() { var r RackController var ifs, dif []byte - if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { + if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered, &r.ServerCA); err != nil { return nil, err } _ = json.Unmarshal(ifs, &r.Interfaces) @@ -465,6 +477,9 @@ func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { st := &rackState{Gen: gen} if rc, ok := a.racksSnapshot()[name]; ok { st.Update = a.rackUpdateFor(rc) + if a.ca != nil && rc.Kind == "remote" { + st.Trust = a.trustState() + } } base := a.rackURL(name) app := a.appendFor(base) @@ -856,6 +871,7 @@ func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack st fail(w, 500, err) return } + a.noteRackTrust(r, rack, rc) a.reloadRacks(r.Context()) a.hub.bump() log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces)) diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 2c1c041..f84d3b5 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -476,9 +476,8 @@ func runRack(args []string) error { mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version}) }) - mux.HandleFunc("GET /ca.pem", func(w http.ResponseWriter, req *http.Request) { // agents check it by its fingerprint - http.ServeFile(w, req, cfg.CAFile) - }) + mux.HandleFunc("GET /ca.pem", r.serveCA(false)) // agents check it by its fingerprint + mux.HandleFunc("GET /ca-bundle.pem", r.serveCA(true)) if r.cert != nil && cfg.TLSListen != "off" { go r.serveTLS(logging(mux)) } @@ -534,8 +533,10 @@ func (r *rackd) register() error { r.srvMu.Lock() tlsURL := r.tlsURL r.srvMu.Unlock() + bundle, serverCA := r.trustReport() _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, TLSURL: tlsURL, Version: version, Arch: runtime.GOARCH, - Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil) + Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot, + CABundleSHA256: bundle, ServerCA: serverCA}, nil) return err } @@ -571,6 +572,7 @@ func (r *rackd) loop() { continue } go r.maybeUpdate(st.Update) + go r.maybeTrust(st.Trust) if st.Gen == r.gen { continue } @@ -790,6 +792,12 @@ func (r *rackd) serverCert() (*tls.Certificate, error) { } } } + return r.newServerCert() +} + +// newServerCert asks the region for a server certificate and keeps it. +func (r *rackd) newServerCert() (*tls.Certificate, error) { + p := r.cfg.serverCertPath() key, csr, err := newCSR(rackServerCN + r.cfg.Name) if err != nil { return nil, err diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index e3a2897..537e83d 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -4,7 +4,6 @@ import ( "context" "crypto/subtle" "crypto/tls" - "crypto/x509" "embed" "encoding/json" "errors" @@ -252,6 +251,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs)) mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete)) mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM)) + mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation)) mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS)) mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync)) mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts)) @@ -269,6 +269,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete)) mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP)) mux.HandleFunc("GET /ca.pem", a.handleCAPEM) + mux.HandleFunc("GET /ca-bundle.pem", a.handleCABundle) mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode)) mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew)) @@ -867,8 +868,7 @@ func serve(cfgPath string) error { if err != nil { return fmt.Errorf("region certificate: %w", err) } - pool := x509.NewCertPool() - pool.AddCert(app.ca.Cert) + pool := app.ca.pool() // a rotation's other root too: racks and admins keep working tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}} go func() { diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index 128a0b0..115dc03 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -473,7 +473,7 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) { } caPEM := "" if a.ca != nil { - caPEM = string(a.ca.PEM) + caPEM = string(a.ca.TrustPEM) // during a root rotation both } return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, CA: caPEM, Vars: vars, Server: a.urlFor(m), Now: time.Now().Format(time.RFC3339)}, nil diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index cfa59e9..381d015 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -170,6 +170,9 @@ ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS ca_bundle_sha256 text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS client_ca text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS server_ca text; ALTER TABLE vlans ADD COLUMN IF NOT EXISTS dhcp_mode text NOT NULL DEFAULT ''; CREATE TABLE IF NOT EXISTS join_tokens ( hash text PRIMARY KEY, diff --git a/deploy/rollcall-rack-update.path b/deploy/rollcall-rack-update.path index 1447554..a0146fc 100644 --- a/deploy/rollcall-rack-update.path +++ b/deploy/rollcall-rack-update.path @@ -1,8 +1,9 @@ [Unit] -Description=rollcall: install a rollcall binary the rack controller staged +Description=rollcall: install a rollcall binary or trust bundle the rack controller staged [Path] PathExists=/var/lib/rollcall/update/ready +PathExists=/var/lib/rollcall/update/ca-ready [Install] WantedBy=multi-user.target diff --git a/deploy/rollcall-rack-update.sh b/deploy/rollcall-rack-update.sh index ef370ce..6cef50c 100755 --- a/deploy/rollcall-rack-update.sh +++ b/deploy/rollcall-rack-update.sh @@ -1,17 +1,18 @@ #!/bin/sh -# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path). +# Installs what the rack controller staged (rollcall-rack-update.path): a new +# trust bundle (root rotation) and a new rollcall binary. # # The staging directory belongs to the rack's unprivileged user, so nothing in -# it is trusted: the binary is copied into a directory of root's and installed -# only if the region's signature over it verifies against the CA certificate -# written at join (root's, from rack.json ca_file). Then the running binary is -# kept as .prev, the rack restarted, and the previous binary put back if the -# rack does not register with the region in 60 s. +# it is trusted: files are copied into a directory of root's and installed only +# if the region's signature over them verifies against one of the roots in the +# CA file (root's, from rack.json ca_file; written at join and by this script). +# A bundle replaces the CA file and restarts the rack. A binary is kept as +# .prev, the rack restarted, and the previous binary put back if the rack does +# not register with the region in 60 s. set -u D=/var/lib/rollcall/update BIN=/usr/local/bin/rollcall -[ -f "$D/ready" ] || exit 0 -want=$(head -c 64 "$D/ready" | tr -cd 'A-Za-z0-9._+-'); rm -f "$D/ready" +[ -f "$D/ready" ] || [ -f "$D/ca-ready" ] || exit 0 # root's work directory, on a file system that allows executing (/run is noexec) install -d -m 0700 -o root -g root /var/lib/rollcall-update W=$(mktemp -d /var/lib/rollcall-update/w.XXXXXX) || exit 1 @@ -19,11 +20,51 @@ trap 'rm -rf "$W"' EXIT # last-result is read by the rack (backoff); renamed in, so a link planted there is replaced, not followed result() { echo "$(date -Is) $*" > "$W/result" && mv -fT "$W/result" "$D/last-result"; echo "$*"; } CA=$(jq -r '.ca_file // "/etc/rollcall/rack/ca.pem"' /etc/rollcall/rack.json 2>/dev/null) -[ -n "$CA" ] && [ "$(stat -c %u "$CA" 2>/dev/null)" = 0 ] || { result "$want: no root-owned CA certificate ($CA), not installed"; exit 1; } +[ -n "$CA" ] && [ "$(stat -c %u "$CA" 2>/dev/null)" = 0 ] || { result "no root-owned CA certificate ($CA), nothing installed"; exit 1; } +# split PEM certificates of $1 into $2.1, $2.2, ... +splitpem() { awk -v p="$2" '/-----BEGIN CERTIFICATE-----/{n++; f=p "." n} f{print > f} /-----END CERTIFICATE-----/{close(f); f=""}' "$1"; } +# signed FILE SIG: the signature verifies against one of the roots in $CA +signed() { + rm -f "$W"/root.*; splitpem "$CA" "$W/root" + for c in "$W"/root.*; do + [ -f "$c" ] || continue + openssl x509 -in "$c" -pubkey -noout > "$W/pub" 2>/dev/null && + openssl dgst -sha256 -verify "$W/pub" -signature "$2" "$1" >/dev/null 2>&1 && return 0 + done + return 1 +} + +if [ -f "$D/ca-ready" ]; then + rm -f "$D/ca-ready" + if cp -- "$D/ca-bundle.pem" "$W/bundle" && cp -- "$D/ca-bundle.sig" "$W/bundle.sig"; then + if ! signed "$W/bundle" "$W/bundle.sig"; then + result "trust bundle: signature does not verify against $CA, not installed" + else + # only CA certificates, rewritten from what was parsed + rm -f "$W"/new.*; splitpem "$W/bundle" "$W/new"; : > "$W/ca.new"; n=0; ok=1 + for c in "$W"/new.*; do + [ -f "$c" ] || continue + openssl x509 -in "$c" -noout -text 2>/dev/null | grep -q "CA:TRUE" || ok= + openssl x509 -in "$c" >> "$W/ca.new" 2>/dev/null || ok= + n=$((n+1)) + done + if [ -z "$ok" ] || [ $n -eq 0 ]; then + result "trust bundle: not CA certificates only, not installed" + else + cp -p "$CA" "$CA.prev" + install -m 0644 -o root -g root "$W/ca.new" "$CA.tmp" && mv -f "$CA.tmp" "$CA" + rm -f "$D/ca-bundle.pem" "$D/ca-bundle.sig" + systemctl restart rollcall-rack.service + result "trust bundle installed ($n roots)" + fi + fi + fi +fi + +[ -f "$D/ready" ] || exit 0 +want=$(head -c 64 "$D/ready" | tr -cd 'A-Za-z0-9._+-'); rm -f "$D/ready" cp -- "$D/rollcall.new" "$W/bin" && cp -- "$D/rollcall.new.sig" "$W/sig" || { result "$want: staged files missing, not installed"; exit 1; } -openssl x509 -in "$CA" -pubkey -noout > "$W/pub" 2>/dev/null && - openssl dgst -sha256 -verify "$W/pub" -signature "$W/sig" "$W/bin" >/dev/null 2>&1 || - { result "$want: signature does not verify against $CA, not installed"; exit 1; } +signed "$W/bin" "$W/sig" || { result "$want: signature does not verify against $CA, not installed"; exit 1; } chmod 0755 "$W/bin" got=$("$W/bin" version 2>/dev/null | awk '{print $2}') [ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; }