From a25b5c1f3379e16977088638ecb40eee5598588c Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 31 Oct 2026 17:36:24 +0100 Subject: [PATCH] Rotation: refresh says when it is too late, admin certificates noted rollcall-ca-refresh reports a server it no longer trusts (the rotation went past prepare before the machine refreshed) instead of exiting quietly; the README asks for a day between prepare and switch. The status lists admin certificates from before the switch, and retire prints them: they stop working with the old root. Tested end to end in the lab: prepare, switch and retire on pxe-server with pxe-rack2 following without a new join (bundle installed by its root unit, client and server certificates renewed), node001 (Debian) and node002 (Rocky) moving to the new root with the refresh timer, and the netops Caddy intermediate signed again by the new root. Closes #12 Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++---- cmd/rollcall/carot.go | 26 ++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d6ae315..ef9c02d 100644 --- a/README.md +++ b/README.md @@ -580,11 +580,16 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and join again). Machines deployed by this version get the roots one per file and a daily `rollcall-ca-refresh` timer that fetches the bundle over HTTPS from their rack, checked against the roots they trust, so they pick up a - new root during `prepare` and drop the old one after `retire`. Machines - deployed earlier need the bundle installed once by hand - (`curl --cacert https:///ca-bundle.pem`). Admin + new root during `prepare` and drop the old one after `retire`: leave at + least a day between `prepare` and `switch` (the rotation status cannot see + deployed machines). A machine that missed it says so in its journal and + needs the bundle by hand. Machines deployed earlier need it once, during + `prepare` (`curl --cacert https:///ca-bundle.pem`; after + `switch` the servers no longer show the old root). Admin certificates and intermediates signed by the old root work until `retire`; - issue them again after `switch`. `/ca.pem` stays a single certificate (the + issue them again after `switch` (the status lists the admin certificates; + an ACME server under an intermediate also needs its stored certificates + issued again, e.g. Caddy's `certificates/` directory moved away). `/ca.pem` stays a single certificate (the root of that server's own certificate), which agents check by fingerprint. ## Power drivers diff --git a/cmd/rollcall/carot.go b/cmd/rollcall/carot.go index a7b1ce2..efaff7f 100644 --- a/cmd/rollcall/carot.go +++ b/cmd/rollcall/carot.go @@ -206,10 +206,12 @@ type rotationStatus struct { Racks []rackTrust `json:"racks"` // what blocks the next step NotReady []string `json:"not_ready"` + // what stops working at retire but does not block it + Warnings []string `json:"warnings"` } func (a *App) rotation(ctx context.Context) (rotationStatus, error) { - st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}} + st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}, Warnings: []string{}} sum := sha256.Sum256(a.ca.TrustPEM) st.Bundle = hex.EncodeToString(sum[:]) if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-next.pem")); err == nil { @@ -220,6 +222,16 @@ func (a *App) rotation(ctx context.Context) (rotationStatus, error) { if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil { if c, err := parseCertPEM(b); err == nil { st.Phase, st.Prev = "switched", fpHex(c) + // admin certificates from before the switch come from the old root + if fi, err := os.Stat(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil { + if cs, err := a.store.AdminCerts(ctx); err == nil { + for _, x := range cs { + if x.Revoked == nil && time.Now().Before(x.Expires) && x.Created.Before(fi.ModTime()) { + st.Warnings = append(st.Warnings, fmt.Sprintf("admin certificate %s (serial %s) is from the old root: rollcall admin-cert request %s", x.Name, x.Serial, x.Name)) + } + } + } + } } } racks, err := a.store.RackTrust(ctx) @@ -312,7 +324,11 @@ func caRefreshFiles(tlsBase string) []seedFile { set -eu URL=` + shQuote(tlsBase+"/ca-bundle.pem") + ` T=$(mktemp -d); trap 'rm -rf "$T"' EXIT -curl -fsS --max-time 30 -o "$T/bundle" "$URL" || exit 0 # unreachable: the timer tries again +rc=0; curl -fsS --max-time 30 -o "$T/bundle" "$URL" || rc=$? +if [ $rc -eq 60 ]; then # the server's certificate is from a root not trusted here + echo "rollcall: $URL is not trusted here: the root rotation went past prepare before this machine refreshed; install the bundle by hand" >&2; exit 1 +fi +[ $rc -eq 0 ] || exit 0 # unreachable: the timer tries again awk -v d="$T" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n)} /-----END CERTIFICATE-----/{close(d "/c" n)}' "$T/bundle" [ -f "$T/c1" ] || exit 0 changed= @@ -421,6 +437,9 @@ func caRotateCmd(cfg *Config, args []string) error { for _, n := range st.NotReady { fmt.Println("waiting:", n) } + for _, n := range st.Warnings { + fmt.Println("note:", n) + } return nil case "prepare": if have("ca-next.pem") || have("ca-prev.pem") { @@ -465,6 +484,9 @@ func caRotateCmd(cfg *Config, args []string) error { if err := ready(); err != nil { return err } + if st, err := status(); err == nil && len(st.Warnings) > 0 { + fmt.Printf("stops working now (intermediates from the old root too):\n %s\n", strings.Join(st.Warnings, "\n ")) + } bak := filepath.Join(dir, "retired-"+time.Now().Format("20060102-150405")+".pem") if err := os.Rename(filepath.Join(dir, "ca-prev.pem"), bak); err != nil { return err