Rotation: refresh says when it is too late, admin certificates noted

rollcall-ca-refresh reports a server it no longer trusts (the rotation
went past prepare before the machine refreshed) instead of exiting
quietly; the README asks for a day between prepare and switch. The
status lists admin certificates from before the switch, and retire
prints them: they stop working with the old root.

Tested end to end in the lab: prepare, switch and retire on pxe-server
with pxe-rack2 following without a new join (bundle installed by its
root unit, client and server certificates renewed), node001 (Debian) and
node002 (Rocky) moving to the new root with the refresh timer, and the
netops Caddy intermediate signed again by the new root.

Closes #12

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -580,11 +580,16 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and
   join again). Machines deployed by this version get the roots one per file
   and a daily `rollcall-ca-refresh` timer that fetches the bundle over HTTPS
   from their rack, checked against the roots they trust, so they pick up a
-  new root during `prepare` and drop the old one after `retire`. Machines
-  deployed earlier need the bundle installed once by hand
-  (`curl --cacert <old root> https://<rack>/ca-bundle.pem`). Admin
+  new root during `prepare` and drop the old one after `retire`: leave at
+  least a day between `prepare` and `switch` (the rotation status cannot see
+  deployed machines). A machine that missed it says so in its journal and
+  needs the bundle by hand. Machines deployed earlier need it once, during
+  `prepare` (`curl --cacert <old root> https://<rack>/ca-bundle.pem`; after
+  `switch` the servers no longer show the old root). Admin
   certificates and intermediates signed by the old root work until `retire`;
-  issue them again after `switch`. `/ca.pem` stays a single certificate (the
+  issue them again after `switch` (the status lists the admin certificates;
+  an ACME server under an intermediate also needs its stored certificates
+  issued again, e.g. Caddy's `certificates/<ca>` directory moved away). `/ca.pem` stays a single certificate (the
   root of that server's own certificate), which agents check by fingerprint.
 
 ## Power drivers

cmd/rollcall/carot.go

@@ -206,10 +206,12 @@ type rotationStatus struct {
 	Racks  []rackTrust `json:"racks"`
 	// what blocks the next step
 	NotReady []string `json:"not_ready"`
+	// what stops working at retire but does not block it
+	Warnings []string `json:"warnings"`
 }
 
 func (a *App) rotation(ctx context.Context) (rotationStatus, error) {
-	st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}}
+	st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}, Warnings: []string{}}
 	sum := sha256.Sum256(a.ca.TrustPEM)
 	st.Bundle = hex.EncodeToString(sum[:])
 	if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-next.pem")); err == nil {
@@ -220,6 +222,16 @@ func (a *App) rotation(ctx context.Context) (rotationStatus, error) {
 	if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil {
 		if c, err := parseCertPEM(b); err == nil {
 			st.Phase, st.Prev = "switched", fpHex(c)
+			// admin certificates from before the switch come from the old root
+			if fi, err := os.Stat(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil {
+				if cs, err := a.store.AdminCerts(ctx); err == nil {
+					for _, x := range cs {
+						if x.Revoked == nil && time.Now().Before(x.Expires) && x.Created.Before(fi.ModTime()) {
+							st.Warnings = append(st.Warnings, fmt.Sprintf("admin certificate %s (serial %s) is from the old root: rollcall admin-cert request %s", x.Name, x.Serial, x.Name))
+						}
+					}
+				}
+			}
 		}
 	}
 	racks, err := a.store.RackTrust(ctx)
@@ -312,7 +324,11 @@ func caRefreshFiles(tlsBase string) []seedFile {
 set -eu
 URL=` + shQuote(tlsBase+"/ca-bundle.pem") + `
 T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
-curl -fsS --max-time 30 -o "$T/bundle" "$URL" || exit 0 # unreachable: the timer tries again
+rc=0; curl -fsS --max-time 30 -o "$T/bundle" "$URL" || rc=$?
+if [ $rc -eq 60 ]; then # the server's certificate is from a root not trusted here
+  echo "rollcall: $URL is not trusted here: the root rotation went past prepare before this machine refreshed; install the bundle by hand" >&2; exit 1
+fi
+[ $rc -eq 0 ] || exit 0 # unreachable: the timer tries again
 awk -v d="$T" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n)} /-----END CERTIFICATE-----/{close(d "/c" n)}' "$T/bundle"
 [ -f "$T/c1" ] || exit 0
 changed=
@@ -421,6 +437,9 @@ func caRotateCmd(cfg *Config, args []string) error {
 		for _, n := range st.NotReady {
 			fmt.Println("waiting:", n)
 		}
+		for _, n := range st.Warnings {
+			fmt.Println("note:", n)
+		}
 		return nil
 	case "prepare":
 		if have("ca-next.pem") || have("ca-prev.pem") {
@@ -465,6 +484,9 @@ func caRotateCmd(cfg *Config, args []string) error {
 		if err := ready(); err != nil {
 			return err
 		}
+		if st, err := status(); err == nil && len(st.Warnings) > 0 {
+			fmt.Printf("stops working now (intermediates from the old root too):\n  %s\n", strings.Join(st.Warnings, "\n  "))
+		}
 		bak := filepath.Join(dir, "retired-"+time.Now().Format("20060102-150405")+".pem")
 		if err := os.Rename(filepath.Join(dir, "ca-prev.pem"), bak); err != nil {
 			return err