ca intermediate: the renewal hint names the region's HTTPS address, not loopback

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -6,6 +6,15 @@ publish a tag that has no section here.
 
 ## [Unreleased]
 
+### Added
+- Intermediates renew themselves: the region keeps the intermediates it signs (`rollcall ca
+  intermediate` goes through the API now and prints a renewal token once; `rollcall ca
+  intermediates` lists them), and `deploy/rollcall-intermediate-renew.sh` with its daily timer on
+  the intermediate's host gets a new certificate in the last third of its life or after a root
+  rotation, always with the first name constraints. An intermediate near its end, failing to renew
+  or whose host went silent shows in `rollcall doctor`, on the overview and in the rotation status.
+  (#14)
+
 ## [0.4.0] - 2026-11-01
 
 ### Added

README.md

@@ -587,6 +587,34 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and
   can be issued. Roots created before this (path length 0) cannot delegate:
   rotate the root (below). `deploy/tags/acme-cert.yaml` is a tag that fetches
   a node certificate at first boot and renews it.
+* **Intermediates renew themselves.** The region keeps every intermediate it
+  signs and prints a renewal token once (`NAME.token` next to `NAME.pem`). On
+  the host with the intermediate's key, `deploy/rollcall-intermediate-renew.sh`
+  (installed as `/usr/local/sbin/rollcall-intermediate-renew`) and the
+  `rollcall-intermediate-renew@NAME.timer` ask the region every day with that
+  token and a CSR from the same key; settings in
+  `/etc/rollcall-intermediate/NAME.env`:
+
+  ```
+  URL=https://region.example
+  TOKEN_FILE=/etc/rollcall-intermediate/caddy.token
+  CERT=/etc/caddy/pki/int.pem
+  KEY=/etc/caddy/pki/int-key.pem
+  ROOT=/etc/caddy/pki/root.pem          # optional: kept to the active root
+  RELOAD='systemctl restart caddy'      # optional: after a new certificate
+  ```
+
+  The region signs again in the last third of the certificate's life, or right
+  away when another root signs since a rotation's `switch`, always with the
+  name constraints it was first given; the script checks what comes back (a CA
+  certificate for its key, from a root it holds), keeps the old file as
+  `.prev`, keeps the region's roots current from `/ca-bundle.pem`, and runs
+  `RELOAD`. `rollcall ca intermediates` lists them; one under 30 days from its
+  end, whose renewal failed or whose host has not asked for three days shows in
+  `rollcall doctor`, on the web UI's overview and in the rotation status.
+  `-replace` signs one the region keeps again with a new token (a lost token);
+  `DELETE /api/v1/ca/intermediates?name=` stops keeping it (it stays valid to
+  its end: there is no revocation list).
 * **Root rotation without re-joining.** In three steps, each followed by a
   restart of the region (`systemctl restart rollcall`):
 

cmd/rollcall/ca.go

@@ -684,7 +684,9 @@ func caCmd(args []string) error {
 	}
 	switch {
 	case args[0] == "intermediate" && len(args) >= 2:
-		return caIntermediateCmd(cfg, args[1], args[2:])
+		return caIntermediateCmd(args[1], args[2:])
+	case args[0] == "intermediates":
+		return caIntermediatesCmd()
 	case args[0] == "rotate":
 		return caRotateCmd(cfg, args[1:])
 	case args[0] != "issue" || len(args) < 2:
@@ -723,76 +725,6 @@ func caCmd(args []string) error {
 	return nil
 }
 
-// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR]
-func caIntermediateCmd(cfg *Config, name string, args []string) error {
-	fs := newFlags("ca intermediate")
-	permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)")
-	csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated")
-	days := fs.Int("days", 365, "validity in days")
-	out := fs.String("out", ".", "directory for NAME.pem (+ NAME-key.pem without -csr)")
-	unconstrained := fs.Bool("unconstrained", false, "allow an intermediate without name constraints")
-	_ = fs.Parse(args)
-	dns, nets, err := parsePermits(*permit)
-	if err != nil {
-		return err
-	}
-	if len(dns)+len(nets) == 0 && !*unconstrained {
-		return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16); -unconstrained to override")
-	}
-	ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA")
-	if err != nil {
-		return err
-	}
-	var pub crypto.PublicKey
-	var keyPEM []byte
-	if *csrFile != "" {
-		b, err := os.ReadFile(*csrFile)
-		if err != nil {
-			return err
-		}
-		blk, _ := pem.Decode(b)
-		if blk == nil {
-			return fmt.Errorf("%s: not PEM", *csrFile)
-		}
-		csr, err := x509.ParseCertificateRequest(blk.Bytes)
-		if err != nil {
-			return err
-		}
-		if err := csr.CheckSignature(); err != nil {
-			return fmt.Errorf("CSR signature: %w", err)
-		}
-		pub = csr.PublicKey
-	} else {
-		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-		if err != nil {
-			return err
-		}
-		pub = &key.PublicKey
-		kder, _ := x509.MarshalECPrivateKey(key)
-		keyPEM = pemEncode("EC PRIVATE KEY", kder)
-	}
-	certPEM, cert, err := ca.intermediate(pub, name, dns, nets, time.Duration(*days)*24*time.Hour)
-	if err != nil {
-		return err
-	}
-	if keyPEM != nil {
-		if err := os.WriteFile(filepath.Join(*out, name+"-key.pem"), keyPEM, 0o600); err != nil {
-			return err
-		}
-	}
-	if err := os.WriteFile(filepath.Join(*out, name+".pem"), certPEM, 0o644); err != nil {
-		return err
-	}
-	var permits []string
-	permits = append(permits, dns...)
-	for _, n := range nets {
-		permits = append(permits, n.String())
-	}
-	fmt.Printf("wrote %s.pem in %s: intermediate CA %q until %s, may issue for %s\nroot: %s (%s)\n", name, *out, name,
-		cert.NotAfter.Format("2006-01-02"), orDash(strings.Join(permits, " ")), filepath.Join(cfg.CADir, "ca.pem"), ca.Fingerprint())
-	return nil
-}
-
 // fileOwner returns uid, gid of path (unix).
 func fileOwner(path string) ([2]int, bool) {
 	st, err := os.Stat(path)

cmd/rollcall/carot.go

@@ -234,6 +234,13 @@ func (a *App) rotation(ctx context.Context) (rotationStatus, error) {
 			}
 		}
 	}
+	if xs, err := a.store.Intermediates(ctx); err == nil { // their hosts renew them from the new root on their own
+		for _, x := range xs {
+			if x.Issuer != st.Active {
+				st.Warnings = append(st.Warnings, fmt.Sprintf("intermediate %s is from the old root: its host renews it on its next daily request (or now: rollcall-intermediate-renew -force there)", x.Name))
+			}
+		}
+	}
 	racks, err := a.store.RackTrust(ctx)
 	if err != nil {
 		return st, err

cmd/rollcall/doctor.go

@@ -83,6 +83,24 @@ func doctorCmd(args []string) error {
 		d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
 	} else {
 		d.ok("database", "connected, schema current")
+		// delegated intermediates the region renews: ending, failing, or a silent host
+		active := ""
+		if b, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
+			if c, err := parseCertPEM(b); err == nil {
+				active = fpHex(c)
+			}
+		}
+		ictx, icancel := context.WithTimeout(context.Background(), 10*time.Second)
+		if xs, err := st.Intermediates(ictx); err == nil {
+			for _, x := range xs {
+				if p := x.problem(active, time.Now()); p != "" {
+					d.warn("intermediate "+x.Name, p, "on its host: systemctl status rollcall-intermediate-renew; rollcall ca intermediates")
+				} else {
+					d.ok("intermediate "+x.Name, "until "+x.NotAfter.Format("2006-01-02")+", renewed by its host")
+				}
+			}
+		}
+		icancel()
 		st.pool.Close()
 	}
 

cmd/rollcall/intermediates.go

@@ -0,0 +1,425 @@
+package main
+
+// Delegated intermediates kept current. The region keeps every intermediate
+// it signs (its name constraints, certificate and a renewal token shown
+// once), and the host that holds the intermediate's key asks every day with
+// that token and a CSR (POST /api/v1/ca/intermediates/NAME/renew; the
+// deploy/rollcall-intermediate-renew script and timer). The region signs
+// again when the certificate is in the last third of its life or comes from
+// a root that is no longer the active one (after a rotation's switch),
+// always with the constraints it was first given: a renewal never widens
+// what the intermediate may issue. Otherwise it answers that nothing is due.
+//
+// An intermediate under 30 days from its end, whose last renewal failed, or
+// whose host has not asked for three days needs attention: rollcall doctor,
+// GET /api/v1/ca/intermediates, the web UI's overview, and the rotation
+// status for one from the old root.
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"text/tabwriter"
+	"time"
+)
+
+type Intermediate struct {
+	Name      string     `json:"name"`
+	Permits   []string   `json:"permits"`
+	Days      int        `json:"days"`
+	NotAfter  time.Time  `json:"not_after"`
+	Issuer    string     `json:"issuer"` // fingerprint of the root that signed it
+	Created   time.Time  `json:"created"`
+	Renewed   *time.Time `json:"renewed,omitempty"`
+	LastSeen  *time.Time `json:"last_seen,omitempty"` // the host's last renewal request
+	LastError string     `json:"last_error,omitempty"`
+	Problem   string     `json:"problem,omitempty"`
+	cert      string
+	tokenHash string
+}
+
+const (
+	interWarnLeft = 30 * 24 * time.Hour // under this to its end: attention
+	interSilent   = 72 * time.Hour      // the host has not asked for this long: attention
+)
+
+// problem: what needs attention about x, "" when nothing does
+func (x *Intermediate) problem(activeRoot string, now time.Time) string {
+	var p []string
+	if left := x.NotAfter.Sub(now); left < interWarnLeft {
+		if left <= 0 {
+			p = append(p, "expired "+x.NotAfter.Format("2006-01-02"))
+		} else {
+			p = append(p, fmt.Sprintf("expires in %d days (%s)", int(math.Ceil(left.Hours()/24-1e-9)), x.NotAfter.Format("2006-01-02")))
+		}
+	}
+	if x.LastError != "" {
+		p = append(p, "last renewal failed: "+x.LastError)
+	}
+	switch {
+	case x.LastSeen == nil && now.Sub(x.Created) > interSilent:
+		p = append(p, "its host never asked for renewal: install deploy/rollcall-intermediate-renew there")
+	case x.LastSeen != nil && now.Sub(*x.LastSeen) > interSilent:
+		p = append(p, "its host has not asked since "+x.LastSeen.Format("2006-01-02 15:04"))
+	}
+	if activeRoot != "" && x.Issuer != activeRoot {
+		p = append(p, "signed by a root that no longer signs: renewed on its host's next request")
+	}
+	return strings.Join(p, "; ")
+}
+
+// due: whether a renewal request gets a new certificate
+func (x *Intermediate) due(activeRoot string, now time.Time) bool {
+	c, err := parseCertPEM([]byte(x.cert))
+	if err != nil {
+		return true
+	}
+	life := c.NotAfter.Sub(c.NotBefore)
+	return c.NotAfter.Sub(now) < life/3 || x.Issuer != activeRoot
+}
+
+// ---------------------------------------------------------------- store
+
+const interCols = `name, permits, days, cert, not_after, issuer, created, renewed, last_seen, coalesce(last_error,''), token_hash`
+
+func scanInter(row interface{ Scan(...any) error }) (*Intermediate, error) {
+	var x Intermediate
+	var permits string
+	if err := row.Scan(&x.Name, &permits, &x.Days, &x.cert, &x.NotAfter, &x.Issuer, &x.Created, &x.Renewed, &x.LastSeen, &x.LastError, &x.tokenHash); err != nil {
+		return nil, err
+	}
+	x.Permits = splitList(permits)
+	return &x, nil
+}
+
+func (s *Store) Intermediates(ctx context.Context) ([]*Intermediate, error) {
+	rows, err := s.pool.Query(ctx, `SELECT `+interCols+` FROM intermediates ORDER BY name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []*Intermediate{}
+	for rows.Next() {
+		x, err := scanInter(rows)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, x)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) Intermediate(ctx context.Context, name string) (*Intermediate, error) {
+	x, err := scanInter(s.pool.QueryRow(ctx, `SELECT `+interCols+` FROM intermediates WHERE name=$1`, name))
+	if err != nil && strings.Contains(err.Error(), "no rows") {
+		return nil, errNotFound
+	}
+	return x, err
+}
+
+func (s *Store) PutIntermediate(ctx context.Context, x *Intermediate) error {
+	_, err := s.pool.Exec(ctx, `INSERT INTO intermediates(name, permits, days, cert, not_after, issuer, token_hash) VALUES ($1,$2,$3,$4,$5,$6,$7)
+		ON CONFLICT (name) DO UPDATE SET permits=$2, days=$3, cert=$4, not_after=$5, issuer=$6, token_hash=$7, created=now(), renewed=NULL, last_seen=NULL, last_error=NULL`,
+		x.Name, strings.Join(x.Permits, ","), x.Days, x.cert, x.NotAfter, x.Issuer, x.tokenHash)
+	return err
+}
+
+// RenewedIntermediate stores a renewal (or, with cert "", only that the host asked, and how it went).
+func (s *Store) RenewedIntermediate(ctx context.Context, name, cert string, notAfter time.Time, issuer, lastErr string) error {
+	if cert == "" {
+		_, err := s.pool.Exec(ctx, `UPDATE intermediates SET last_seen=now(), last_error=nullif($2,'') WHERE name=$1`, name, lastErr)
+		return err
+	}
+	_, err := s.pool.Exec(ctx, `UPDATE intermediates SET cert=$2, not_after=$3, issuer=$4, renewed=now(), last_seen=now(), last_error=NULL WHERE name=$1`, name, cert, notAfter, issuer)
+	return err
+}
+
+func (s *Store) DeleteIntermediate(ctx context.Context, name string) error {
+	_, err := s.pool.Exec(ctx, `DELETE FROM intermediates WHERE name=$1`, name)
+	return err
+}
+
+// ---------------------------------------------------------------- signing
+
+// csrKey: the public key of a PEM CSR whose signature holds
+func csrKey(b []byte) (any, error) {
+	blk, _ := pem.Decode(b)
+	if blk == nil || blk.Type != "CERTIFICATE REQUEST" && blk.Type != "NEW CERTIFICATE REQUEST" {
+		return nil, errors.New("not a PEM certificate request")
+	}
+	csr, err := x509.ParseCertificateRequest(blk.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	if err := csr.CheckSignature(); err != nil {
+		return nil, fmt.Errorf("CSR signature: %w", err)
+	}
+	return csr.PublicKey, nil
+}
+
+func (a *App) signIntermediate(name string, permits []string, days int, csrPEM []byte) (string, *x509.Certificate, error) {
+	pub, err := csrKey(csrPEM)
+	if err != nil {
+		return "", nil, err
+	}
+	dns, nets, err := parsePermits(strings.Join(permits, ","))
+	if err != nil {
+		return "", nil, err
+	}
+	certPEM, cert, err := a.ca.intermediate(pub, name, dns, nets, time.Duration(days)*24*time.Hour)
+	return string(certPEM), cert, err
+}
+
+var interNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`)
+
+// splitList: the items of a comma-separated list, trimmed, without empty ones
+func splitList(s string) []string {
+	var out []string
+	for _, x := range strings.Split(s, ",") {
+		if x = strings.TrimSpace(x); x != "" {
+			out = append(out, x)
+		}
+	}
+	return out
+}
+
+func hashToken(t string) string { s := sha256.Sum256([]byte(t)); return hex.EncodeToString(s[:]) }
+
+// ---------------------------------------------------------------- handlers
+
+// handleIntermediates: GET lists them with what needs attention; POST signs
+// one ({"name","permits","csr","days","replace"}) and returns its
+// certificate, the root and the renewal token (shown this once); DELETE
+// ?name= stops keeping it (it stays valid until its end: there is no CRL).
+func (a *App) handleIntermediates(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	switch r.Method {
+	case http.MethodGet:
+		xs, err := a.store.Intermediates(ctx)
+		if err != nil {
+			fail(w, 500, err)
+			return
+		}
+		for _, x := range xs {
+			x.Problem = x.problem(fpHex(a.ca.Cert), time.Now())
+		}
+		writeJSON(w, 200, xs)
+	case http.MethodDelete:
+		if err := a.store.DeleteIntermediate(ctx, r.URL.Query().Get("name")); err != nil {
+			fail(w, 500, err)
+			return
+		}
+		writeJSON(w, 200, obj{"deleted": r.URL.Query().Get("name")})
+	case http.MethodPost:
+		var req struct {
+			Name, CSR string
+			Permits   []string
+			Days      int
+			Replace   bool
+		}
+		if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil || !interNameRe.MatchString(req.Name) || req.CSR == "" {
+			fail(w, 400, errors.New("name (letters, digits, - . _) and csr are required"))
+			return
+		}
+		if len(req.Permits) == 0 {
+			fail(w, 400, errors.New("an intermediate needs permits (DNS domains and CIDRs it may issue for)"))
+			return
+		}
+		if req.Days == 0 {
+			req.Days = 365
+		}
+		req.Days = max(1, min(req.Days, 1825))
+		if _, err := a.store.Intermediate(ctx, req.Name); err == nil && !req.Replace {
+			fail(w, 409, fmt.Errorf("%s is kept already: replace to sign it again with a new token", req.Name))
+			return
+		}
+		cert, c, err := a.signIntermediate(req.Name, req.Permits, req.Days, []byte(req.CSR))
+		if err != nil {
+			fail(w, 400, err)
+			return
+		}
+		tok := make([]byte, 32)
+		_, _ = rand.Read(tok)
+		token := hex.EncodeToString(tok)
+		x := &Intermediate{Name: req.Name, Permits: req.Permits, Days: req.Days, NotAfter: c.NotAfter, Issuer: fpHex(a.ca.Cert), cert: cert, tokenHash: hashToken(token)}
+		if err := a.store.PutIntermediate(ctx, x); err != nil {
+			fail(w, 500, err)
+			return
+		}
+		writeJSON(w, 200, obj{"name": x.Name, "cert": cert, "root": string(a.ca.PEM), "not_after": x.NotAfter, "token": token})
+	}
+}
+
+// handleIntermediateRenew: the intermediate's host, with its token and a CSR
+// in the body; a new certificate when one is due (?force=1: now), else
+// {"renewed": false}. With Accept: application/x-pem-file the answer is the
+// certificate itself, or 204 when nothing is due (for a shell script). HTTPS only.
+func (a *App) handleIntermediateRenew(w http.ResponseWriter, r *http.Request) {
+	ctx, name := r.Context(), r.PathValue("name")
+	if r.TLS == nil && a.cfg.AdminHTTP != "allow" {
+		host, _, _ := net.SplitHostPort(r.RemoteAddr)
+		if ip := net.ParseIP(host); a.cfg.AdminHTTP != "loopback" || ip == nil || !ip.IsLoopback() {
+			fail(w, 403, errors.New("renewal is HTTPS-only"))
+			return
+		}
+	}
+	x, err := a.store.Intermediate(ctx, name)
+	tok, _ := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+	if err != nil || !tokenEqual(hashToken(tok), x.tokenHash) {
+		fail(w, 401, errors.New("unknown intermediate or wrong token")) // one answer for both: names are not probed
+		return
+	}
+	csr, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<16))
+	if err != nil {
+		fail(w, 400, err)
+		return
+	}
+	active := fpHex(a.ca.Cert)
+	asPEM := strings.Contains(r.Header.Get("Accept"), "application/x-pem-file")
+	if !x.due(active, time.Now()) && r.URL.Query().Get("force") != "1" {
+		_ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", "")
+		if asPEM {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		writeJSON(w, 200, obj{"renewed": false, "not_after": x.NotAfter})
+		return
+	}
+	cert, c, err := a.signIntermediate(x.Name, x.Permits, x.Days, csr) // the constraints it was given, never others
+	if err != nil {
+		_ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", err.Error())
+		fail(w, 400, err)
+		return
+	}
+	if err := a.store.RenewedIntermediate(ctx, name, cert, c.NotAfter, active, ""); err != nil {
+		fail(w, 500, err)
+		return
+	}
+	log.Printf("ca: intermediate %s renewed until %s", name, c.NotAfter.Format("2006-01-02"))
+	if asPEM {
+		w.Header().Set("Content-Type", "application/x-pem-file")
+		_, _ = io.WriteString(w, cert)
+		return
+	}
+	writeJSON(w, 200, obj{"renewed": true, "cert": cert, "root": string(a.ca.PEM), "not_after": c.NotAfter})
+}
+
+// ---------------------------------------------------------------- CLI
+
+// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR] [-replace]
+func caIntermediateCmd(name string, args []string) error {
+	fs := newFlags("ca intermediate")
+	permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)")
+	csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated here")
+	days := fs.Int("days", 365, "validity in days")
+	out := fs.String("out", ".", "directory for NAME.pem, NAME.token (+ NAME-key.pem without -csr)")
+	replace := fs.Bool("replace", false, "sign again an intermediate the region keeps already (a new renewal token)")
+	_ = fs.Parse(args)
+	permits := splitList(*permit)
+	if len(permits) == 0 {
+		return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16)")
+	}
+	if _, _, err := parsePermits(*permit); err != nil {
+		return err
+	}
+	var csrPEM, keyPEM []byte
+	var err error
+	if *csrFile != "" {
+		if csrPEM, err = os.ReadFile(*csrFile); err != nil {
+			return err
+		}
+	} else {
+		var csr string
+		if keyPEM, csr, err = newCSR(name); err != nil {
+			return err
+		}
+		csrPEM = []byte(csr)
+	}
+	c, err := newClient()
+	if err != nil {
+		return err
+	}
+	var res struct {
+		Cert, Root, Token string
+		NotAfter          time.Time `json:"not_after"`
+	}
+	if err := c.do("POST", "/api/v1/ca/intermediates", obj{"name": name, "permits": permits, "csr": string(csrPEM), "days": *days, "replace": *replace}, &res); err != nil {
+		return err
+	}
+	write := func(f string, b []byte, mode os.FileMode) error { return os.WriteFile(filepath.Join(*out, f), b, mode) }
+	if keyPEM != nil {
+		if err := write(name+"-key.pem", keyPEM, 0o600); err != nil {
+			return err
+		}
+	}
+	if err := write(name+".pem", []byte(res.Cert), 0o644); err != nil {
+		return err
+	}
+	if err := write(name+".token", []byte(res.Token+"\n"), 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("wrote %s.pem and %s.token in %s: intermediate CA %q until %s, may issue for %s\n", name, name, *out, name, res.NotAfter.Format("2006-01-02"), strings.Join(permits, " "))
+	fmt.Printf("the region renews it: on the host with its key, install deploy/rollcall-intermediate-renew with\n  URL=%s NAME=%s TOKEN_FILE=<%s.token there> CERT=<the certificate file> KEY=<the key file>\n", regionHTTPS(c.base), name, name)
+	return nil
+}
+
+// rollcall ca intermediates: the intermediates the region keeps
+func caIntermediatesCmd() error {
+	c, err := newClient()
+	if err != nil {
+		return err
+	}
+	var xs []Intermediate
+	if err := c.do("GET", "/api/v1/ca/intermediates", nil, &xs); err != nil {
+		return err
+	}
+	tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(tw, "NAME\tUNTIL\tLAST ASKED\tPERMITS\tATTENTION")
+	for _, x := range xs {
+		seen := "never"
+		if x.LastSeen != nil {
+			seen = x.LastSeen.Format("2006-01-02 15:04")
+		}
+		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", x.Name, x.NotAfter.Format("2006-01-02"), seen, strings.Join(x.Permits, " "), orDash(x.Problem))
+	}
+	return tw.Flush()
+}
+
+// regionHTTPS: the region's HTTPS address for another host, when the CLI
+// talks to it over loopback HTTP on the region itself
+func regionHTTPS(base string) string {
+	if !strings.HasPrefix(base, "http://") {
+		return base
+	}
+	cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"))
+	if err != nil || cfg.TLSListen == "" {
+		return base
+	}
+	u, err := url.Parse(cfg.ServerURL)
+	if err != nil || u.Hostname() == "" {
+		return base
+	}
+	host := u.Hostname()
+	if _, port, _ := net.SplitHostPort(cfg.TLSListen); port != "" && port != "443" {
+		host = net.JoinHostPort(host, port)
+	} else if strings.Contains(host, ":") {
+		host = "[" + host + "]"
+	}
+	return "https://" + host
+}

cmd/rollcall/intermediates_test.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// a renewal signs with the constraints the region keeps, whatever the CSR asks for
+func TestIntermediateRenewalKeepsConstraints(t *testing.T) {
+	ca, err := loadOrCreateCA(t.TempDir(), "test CA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := &App{ca: ca}
+	_, csr, err := newCSR("evil.example.org") // the name in the CSR does not matter
+	if err != nil {
+		t.Fatal(err)
+	}
+	pemCert, c, err := a.signIntermediate("caddy", []string{"int.example.com", "10.213.0.0/16"}, 90, []byte(csr))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !c.IsCA || c.Subject.CommonName != "caddy" || len(c.PermittedDNSDomains) != 1 || c.PermittedDNSDomains[0] != "int.example.com" ||
+		len(c.PermittedIPRanges) != 1 || c.PermittedIPRanges[0].String() != "10.213.0.0/16" || !c.PermittedDNSDomainsCritical {
+		t.Fatalf("constraints: %+v %v %v", c.Subject, c.PermittedDNSDomains, c.PermittedIPRanges)
+	}
+	if d := c.NotAfter.Sub(c.NotBefore); d < 89*24*time.Hour || d > 91*24*time.Hour {
+		t.Fatalf("validity %v", d)
+	}
+	if _, _, err := a.signIntermediate("caddy", []string{"int.example.com"}, 90, []byte("not a csr")); err == nil {
+		t.Fatal("signed something that is not a CSR")
+	}
+
+	// due in the last third of its life, or when another root signs now
+	x := &Intermediate{Name: "caddy", cert: pemCert, Issuer: fpHex(ca.Cert)}
+	if x.due(fpHex(ca.Cert), time.Now()) {
+		t.Fatal("due right after signing")
+	}
+	if !x.due(fpHex(ca.Cert), time.Now().Add(61*24*time.Hour)) {
+		t.Fatal("not due with a third of its life left")
+	}
+	if !x.due("another-root", time.Now()) {
+		t.Fatal("not due after a rotation")
+	}
+}
+
+func TestIntermediateProblems(t *testing.T) {
+	now := time.Now()
+	ago := func(d time.Duration) *time.Time { x := now.Add(-d); return &x }
+	for _, c := range []struct {
+		x    Intermediate
+		want string // a fragment, "" for none
+	}{
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-time.Hour)}, ""},
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour), LastSeen: ago(time.Hour)}, ""},
+		{Intermediate{NotAfter: now.Add(10 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expires in 10 days"},
+		{Intermediate{NotAfter: now.Add(-time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expired"},
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour)}, "never asked"},
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(100 * time.Hour)}, "has not asked since"},
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour), LastError: "CSR signature"}, "last renewal failed"},
+		{Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "old", LastSeen: ago(time.Hour)}, "no longer signs"},
+	} {
+		got := c.x.problem("r", now)
+		if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
+			t.Errorf("%+v: %q, want %q", c.x, got, c.want)
+		}
+	}
+	if hashToken("a") == hashToken("b") || len(hashToken("a")) != 64 {
+		t.Fatal("hashToken")
+	}
+	if got := splitList(" a, ,b ,"); len(got) != 2 || got[0] != "a" || got[1] != "b" {
+		t.Fatalf("splitList %q", got)
+	}
+}

cmd/rollcall/main.go

@@ -32,7 +32,8 @@ func usage() {
   rollcall rack [-config /etc/rollcall/rack.json]   a rack controller next to the machines
   rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80]   enroll a rack controller (writes rack.json + certificate)
   rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR]       the installation's certificate authority (region host)
-  rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365]   delegate to an ACME CA
+  rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365]   delegate to an ACME CA (the region renews it)
+  rollcall ca intermediates                        the intermediates the region keeps, and what needs attention
   rollcall ca rotate [status | prepare | switch | retire] [-force]   replace the root without re-joining racks
   rollcall version
   rollcall keygen

cmd/rollcall/server.go

@@ -252,6 +252,10 @@ func (a *App) routes() http.Handler {
 	mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
 	mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM))
 	mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation))
+	mux.HandleFunc("GET /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates))
+	mux.HandleFunc("POST /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates))
+	mux.HandleFunc("DELETE /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates))
+	mux.HandleFunc("POST /api/v1/ca/intermediates/{name}/renew", a.handleIntermediateRenew) // its own token (intermediates.go)
 	mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS))
 	mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync))
 	mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))

cmd/rollcall/store.go

@@ -115,6 +115,19 @@ CREATE TABLE IF NOT EXISTS switches (
   source   text NOT NULL DEFAULT 'local'
 );
 ALTER TABLE switches ADD COLUMN IF NOT EXISTS role text;
+CREATE TABLE IF NOT EXISTS intermediates (
+  name       text PRIMARY KEY,
+  permits    text NOT NULL,
+  days       int  NOT NULL,
+  cert       text NOT NULL,
+  not_after  timestamptz NOT NULL,
+  issuer     text NOT NULL,
+  token_hash text NOT NULL,
+  created    timestamptz NOT NULL DEFAULT now(),
+  renewed    timestamptz,
+  last_seen  timestamptz,
+  last_error text
+);
 CREATE TABLE IF NOT EXISTS cables (
   machine_id  text NOT NULL REFERENCES machines(id) ON DELETE CASCADE,
   port        text NOT NULL,

cmd/rollcall/web/app.js

@@ -91,7 +91,7 @@ async function refresh(full) {
     $('#n-net').textContent = off ? `${off} offline` : (subnets.length || '');
     const page = route().page;
     if (full || page === 'overview' || page === 'jobs') S.jobs = await api('GET', '/api/v1/jobs');
-    if (full || page === 'overview') S.events = await api('GET', '/api/v1/events');
+    if (full || page === 'overview') [S.events, S.inter] = await Promise.all([api('GET', '/api/v1/events'), api('GET', '/api/v1/ca/intermediates').catch(() => [])]);
     if (full || page === 'sites') [S.sites, S.rooms, S.cabling] = await Promise.all([api('GET', '/api/v1/sites'), api('GET', '/api/v1/rooms'), api('GET', '/api/v1/cabling').catch(() => null)]);
     if (full || page === 'overview' || page === 'network') {
       [S.ips, S.ipam] = await Promise.all([api('GET', '/api/v1/ips'), api('GET', '/api/v1/ipam').catch(() => null)]);
@@ -181,7 +181,10 @@ function pageOverview() {
         ['Jobs failed (last 100)', failed.length ? `<span class="bad">${failed.length}</span>` : '0'], ['Oldest ready', (() => { const r = ready.slice().sort((a, b) => new Date(a.last_seen) - new Date(b.last_seen))[0]; return r ? `${esc(nameOf(r))} · ${ago(r.last_seen)}` : '—'; })()]])}</div></div>`;
 
   // attention
+  const inter = S.inter || [], interBad = inter.filter(x => x.problem);
   const attention = kv([
+    ...(inter.length ? [['Intermediate CAs', interBad.length ? interBad.map(x => `<span class="warn">${esc(x.name)}: ${esc(x.problem)}</span>`).join('<br>')
+      : `<span class="ok">${inter.length} renewed by their hosts</span> <span class="muted">(next end ${esc(inter.map(x => x.not_after).sort()[0].slice(0, 10))})</span>`]] : []),
     ['Rack controllers', offline.length ? `<span class="bad">${offline.map(c => esc(c.name)).join(', ')} offline</span> <a href="#/network">show</a>` : `<span class="ok">${S.controllers.length} online</span>`],
     ['External IPAM', !S.ipam || !S.ipam.provider ? '<span class="muted">built-in only</span>' : ipamLast && ipamLast.error ? `<span class="bad">${esc(ipamLast.error.slice(0, 80))}</span>` : `<span class="ok">${esc(S.ipam.provider)} in sync</span>${ipamLast && ipamLast.conflicts && ipamLast.conflicts.length ? ` <span class="warn">${ipamLast.conflicts.length} conflicts</span>` : ''} <span class="muted">${ipamLast && ipamLast.at ? ago(ipamLast.at) + ' ago' : ''}</span>`],
     ['BMC not configured', bmcBad ? `<span class="bad">${bmcBad}</span> <a href="#/machines">show</a>` : '<span class="ok">none</span>'],

deploy/rollcall-intermediate-renew.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# rollcall-intermediate-renew [-force] NAME: keep a delegated intermediate CA
+# current. Runs daily on the host that holds the intermediate's key (an ACME
+# server: Caddy, step-ca), from /etc/rollcall-intermediate/NAME.env:
+#   URL=https://region.example        the region (HTTPS)
+#   TOKEN_FILE=/etc/rollcall-intermediate/NAME.token   from `rollcall ca intermediate`
+#   CERT=/etc/caddy/pki/int.pem       the intermediate's certificate (replaced)
+#   KEY=/etc/caddy/pki/int-key.pem    its key (stays; the CSR is made from it)
+#   ROOT=/etc/caddy/pki/root.pem      optional: kept to the active root
+#   CA=/etc/rollcall-intermediate/NAME-bundle.pem   the region's roots (seeded from ROOT, kept current)
+#   RELOAD='systemctl restart caddy'  optional: after a new certificate
+# The region decides whether a new certificate is due (the last third of its
+# life, or a root rotation) and signs with the constraints it was given.
+set -eu
+force=
+[ "${1:-}" = -force ] && { force=1; shift; }
+NAME=${1:?usage: rollcall-intermediate-renew [-force] NAME}
+. "/etc/rollcall-intermediate/$NAME.env"
+: "${URL:?}" "${TOKEN_FILE:?}" "${CERT:?}" "${KEY:?}"
+CA=${CA:-/etc/rollcall-intermediate/$NAME-bundle.pem}
+if [ ! -s "$CA" ]; then
+  [ -n "${ROOT:-}" ] && [ -s "$ROOT" ] || { echo "rollcall: $CA missing and no ROOT to start it from" >&2; exit 1; }
+  cp "$ROOT" "$CA"
+fi
+T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
+
+# the region's roots, checked against the ones held: a rotation reaches here during prepare
+if curl -fsS --max-time 30 --cacert "$CA" -o "$T/bundle" "$URL/ca-bundle.pem" && grep -q 'BEGIN CERTIFICATE' "$T/bundle"; then
+  cmp -s "$T/bundle" "$CA" || { cat "$T/bundle" > "$CA"; echo "rollcall: $NAME: the region's roots changed; $CA updated"; }
+fi
+
+openssl req -new -key "$KEY" -subj "/CN=$NAME" -out "$T/csr" 2>/dev/null
+q=; [ -n "$force" ] && q='?force=1'
+code=$(curl -sS --max-time 60 --cacert "$CA" -o "$T/cert" -w '%{http_code}' -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \
+  -H 'Accept: application/x-pem-file' --data-binary @"$T/csr" "$URL/api/v1/ca/intermediates/$NAME/renew$q") || { echo "rollcall: $NAME: the region did not answer" >&2; exit 1; }
+case $code in
+  204) exit 0 ;; # nothing due
+  200) ;;
+  *) echo "rollcall: $NAME: renewal refused (HTTP $code): $(cat "$T/cert")" >&2; exit 1 ;;
+esac
+
+# what came back: a CA certificate for this key, from a root held here
+openssl x509 -in "$T/cert" -noout -text | grep -q 'CA:TRUE' || { echo "rollcall: $NAME: the answer is not a CA certificate" >&2; exit 1; }
+[ "$(openssl x509 -in "$T/cert" -noout -pubkey)" = "$(openssl pkey -in "$KEY" -pubout)" ] || { echo "rollcall: $NAME: the certificate is not for $KEY" >&2; exit 1; }
+openssl verify -CAfile "$CA" "$T/cert" >/dev/null || { echo "rollcall: $NAME: the certificate does not verify against $CA" >&2; exit 1; }
+put() { # $1 new content file, $2 target: same owner and mode, the old one kept as .prev
+  [ -f "$2" ] && cp -p "$2" "$2.prev"
+  if [ -f "$2" ]; then cat "$1" > "$2"; else install -m 0644 "$1" "$2"; fi
+}
+put "$T/cert" "$CERT"
+if [ -n "${ROOT:-}" ]; then # the active root: the first certificate of the bundle
+  awk '/-----BEGIN CERTIFICATE-----/{n++} n==1{print} /-----END CERTIFICATE-----/ && n==1{exit}' "$CA" > "$T/root"
+  cmp -s "$T/root" "$ROOT" || put "$T/root" "$ROOT"
+fi
+echo "rollcall: $NAME renewed: $(openssl x509 -in "$CERT" -noout -enddate)"
+[ -n "${RELOAD:-}" ] && sh -c "$RELOAD"
+exit 0

deploy/rollcall-intermediate-renew@.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=rollcall: renew the delegated intermediate CA %i
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/rollcall-intermediate-renew %i

deploy/rollcall-intermediate-renew@.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=rollcall: renew the delegated intermediate CA %i daily
+
+[Timer]
+OnBootSec=15min
+OnUnitActiveSec=1d
+RandomizedDelaySec=1h
+Persistent=true
+
+[Install]
+WantedBy=timers.target