From 05861bd41315eae5a876e6145d533ecfde11532d Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 1 Nov 2026 17:33:31 +0100 Subject: [PATCH] Intermediates renew themselves through the region The region keeps every intermediate it signs: its name constraints, certificate, the root that signed it and a renewal token (stored hashed, printed once). `rollcall ca intermediate` goes through the API now (so it works from any admin client) and writes NAME.token beside NAME.pem; `rollcall ca intermediates` lists them. The host with the intermediate's key runs deploy/rollcall-intermediate- renew.sh daily (rollcall-intermediate-renew@NAME.timer): it posts a CSR from the same key with the token to /api/v1/ca/intermediates/NAME/renew. The region signs again in the last third of the certificate's life, or at once when another root signs since a rotation's switch, always with the constraints it was first given; otherwise 204. The script checks the answer (a CA certificate for its key, from a root it holds), keeps the old file, keeps the region's roots and the active root current from /ca-bundle.pem, and reloads (RELOAD). Attention: an intermediate under 30 days from its end, whose last renewal failed or whose host has not asked for three days shows in rollcall doctor, on the web UI's overview, and one from the old root in the rotation status. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 + README.md | 28 ++ cmd/rollcall/ca.go | 74 +--- cmd/rollcall/carot.go | 7 + cmd/rollcall/doctor.go | 18 + cmd/rollcall/intermediates.go | 401 ++++++++++++++++++++ cmd/rollcall/intermediates_test.go | 75 ++++ cmd/rollcall/main.go | 3 +- cmd/rollcall/server.go | 4 + cmd/rollcall/store.go | 13 + cmd/rollcall/web/app.js | 5 +- deploy/rollcall-intermediate-renew.sh | 57 +++ deploy/rollcall-intermediate-renew@.service | 8 + deploy/rollcall-intermediate-renew@.timer | 11 + 14 files changed, 640 insertions(+), 73 deletions(-) create mode 100644 cmd/rollcall/intermediates.go create mode 100644 cmd/rollcall/intermediates_test.go create mode 100644 deploy/rollcall-intermediate-renew.sh create mode 100644 deploy/rollcall-intermediate-renew@.service create mode 100644 deploy/rollcall-intermediate-renew@.timer diff --git a/CHANGELOG.md b/CHANGELOG.md index c976e79..2475fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ publish a tag that has no section here. ## [Unreleased] +### Added +- Intermediates renew themselves: the region keeps the intermediates it signs (`rollcall ca + intermediate` goes through the API now and prints a renewal token once; `rollcall ca + intermediates` lists them), and `deploy/rollcall-intermediate-renew.sh` with its daily timer on + the intermediate's host gets a new certificate in the last third of its life or after a root + rotation, always with the first name constraints. An intermediate near its end, failing to renew + or whose host went silent shows in `rollcall doctor`, on the overview and in the rotation status. + (#14) + ## [0.4.0] - 2026-11-01 ### Added diff --git a/README.md b/README.md index 5f67969..72d6c84 100644 --- a/README.md +++ b/README.md @@ -587,6 +587,34 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and can be issued. Roots created before this (path length 0) cannot delegate: rotate the root (below). `deploy/tags/acme-cert.yaml` is a tag that fetches a node certificate at first boot and renews it. +* **Intermediates renew themselves.** The region keeps every intermediate it + signs and prints a renewal token once (`NAME.token` next to `NAME.pem`). On + the host with the intermediate's key, `deploy/rollcall-intermediate-renew.sh` + (installed as `/usr/local/sbin/rollcall-intermediate-renew`) and the + `rollcall-intermediate-renew@NAME.timer` ask the region every day with that + token and a CSR from the same key; settings in + `/etc/rollcall-intermediate/NAME.env`: + + ``` + URL=https://region.example + TOKEN_FILE=/etc/rollcall-intermediate/caddy.token + CERT=/etc/caddy/pki/int.pem + KEY=/etc/caddy/pki/int-key.pem + ROOT=/etc/caddy/pki/root.pem # optional: kept to the active root + RELOAD='systemctl restart caddy' # optional: after a new certificate + ``` + + The region signs again in the last third of the certificate's life, or right + away when another root signs since a rotation's `switch`, always with the + name constraints it was first given; the script checks what comes back (a CA + certificate for its key, from a root it holds), keeps the old file as + `.prev`, keeps the region's roots current from `/ca-bundle.pem`, and runs + `RELOAD`. `rollcall ca intermediates` lists them; one under 30 days from its + end, whose renewal failed or whose host has not asked for three days shows in + `rollcall doctor`, on the web UI's overview and in the rotation status. + `-replace` signs one the region keeps again with a new token (a lost token); + `DELETE /api/v1/ca/intermediates?name=` stops keeping it (it stays valid to + its end: there is no revocation list). * **Root rotation without re-joining.** In three steps, each followed by a restart of the region (`systemctl restart rollcall`): diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index 78751cc..cc8c86f 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -684,7 +684,9 @@ func caCmd(args []string) error { } switch { case args[0] == "intermediate" && len(args) >= 2: - return caIntermediateCmd(cfg, args[1], args[2:]) + return caIntermediateCmd(args[1], args[2:]) + case args[0] == "intermediates": + return caIntermediatesCmd() case args[0] == "rotate": return caRotateCmd(cfg, args[1:]) case args[0] != "issue" || len(args) < 2: @@ -723,76 +725,6 @@ func caCmd(args []string) error { return nil } -// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR] -func caIntermediateCmd(cfg *Config, name string, args []string) error { - fs := newFlags("ca intermediate") - permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)") - csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated") - days := fs.Int("days", 365, "validity in days") - out := fs.String("out", ".", "directory for NAME.pem (+ NAME-key.pem without -csr)") - unconstrained := fs.Bool("unconstrained", false, "allow an intermediate without name constraints") - _ = fs.Parse(args) - dns, nets, err := parsePermits(*permit) - if err != nil { - return err - } - if len(dns)+len(nets) == 0 && !*unconstrained { - return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16); -unconstrained to override") - } - ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - return err - } - var pub crypto.PublicKey - var keyPEM []byte - if *csrFile != "" { - b, err := os.ReadFile(*csrFile) - if err != nil { - return err - } - blk, _ := pem.Decode(b) - if blk == nil { - return fmt.Errorf("%s: not PEM", *csrFile) - } - csr, err := x509.ParseCertificateRequest(blk.Bytes) - if err != nil { - return err - } - if err := csr.CheckSignature(); err != nil { - return fmt.Errorf("CSR signature: %w", err) - } - pub = csr.PublicKey - } else { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return err - } - pub = &key.PublicKey - kder, _ := x509.MarshalECPrivateKey(key) - keyPEM = pemEncode("EC PRIVATE KEY", kder) - } - certPEM, cert, err := ca.intermediate(pub, name, dns, nets, time.Duration(*days)*24*time.Hour) - if err != nil { - return err - } - if keyPEM != nil { - if err := os.WriteFile(filepath.Join(*out, name+"-key.pem"), keyPEM, 0o600); err != nil { - return err - } - } - if err := os.WriteFile(filepath.Join(*out, name+".pem"), certPEM, 0o644); err != nil { - return err - } - var permits []string - permits = append(permits, dns...) - for _, n := range nets { - permits = append(permits, n.String()) - } - fmt.Printf("wrote %s.pem in %s: intermediate CA %q until %s, may issue for %s\nroot: %s (%s)\n", name, *out, name, - cert.NotAfter.Format("2006-01-02"), orDash(strings.Join(permits, " ")), filepath.Join(cfg.CADir, "ca.pem"), ca.Fingerprint()) - return nil -} - // fileOwner returns uid, gid of path (unix). func fileOwner(path string) ([2]int, bool) { st, err := os.Stat(path) diff --git a/cmd/rollcall/carot.go b/cmd/rollcall/carot.go index 6c7d199..f1e99bc 100644 --- a/cmd/rollcall/carot.go +++ b/cmd/rollcall/carot.go @@ -234,6 +234,13 @@ func (a *App) rotation(ctx context.Context) (rotationStatus, error) { } } } + if xs, err := a.store.Intermediates(ctx); err == nil { // their hosts renew them from the new root on their own + for _, x := range xs { + if x.Issuer != st.Active { + st.Warnings = append(st.Warnings, fmt.Sprintf("intermediate %s is from the old root: its host renews it on its next daily request (or now: rollcall-intermediate-renew -force there)", x.Name)) + } + } + } racks, err := a.store.RackTrust(ctx) if err != nil { return st, err diff --git a/cmd/rollcall/doctor.go b/cmd/rollcall/doctor.go index 43d8f75..c1905ec 100644 --- a/cmd/rollcall/doctor.go +++ b/cmd/rollcall/doctor.go @@ -83,6 +83,24 @@ func doctorCmd(args []string) error { d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)") } else { d.ok("database", "connected, schema current") + // delegated intermediates the region renews: ending, failing, or a silent host + active := "" + if b, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + active = fpHex(c) + } + } + ictx, icancel := context.WithTimeout(context.Background(), 10*time.Second) + if xs, err := st.Intermediates(ictx); err == nil { + for _, x := range xs { + if p := x.problem(active, time.Now()); p != "" { + d.warn("intermediate "+x.Name, p, "on its host: systemctl status rollcall-intermediate-renew; rollcall ca intermediates") + } else { + d.ok("intermediate "+x.Name, "until "+x.NotAfter.Format("2006-01-02")+", renewed by its host") + } + } + } + icancel() st.pool.Close() } diff --git a/cmd/rollcall/intermediates.go b/cmd/rollcall/intermediates.go new file mode 100644 index 0000000..872094d --- /dev/null +++ b/cmd/rollcall/intermediates.go @@ -0,0 +1,401 @@ +package main + +// Delegated intermediates kept current. The region keeps every intermediate +// it signs (its name constraints, certificate and a renewal token shown +// once), and the host that holds the intermediate's key asks every day with +// that token and a CSR (POST /api/v1/ca/intermediates/NAME/renew; the +// deploy/rollcall-intermediate-renew script and timer). The region signs +// again when the certificate is in the last third of its life or comes from +// a root that is no longer the active one (after a rotation's switch), +// always with the constraints it was first given: a renewal never widens +// what the intermediate may issue. Otherwise it answers that nothing is due. +// +// An intermediate under 30 days from its end, whose last renewal failed, or +// whose host has not asked for three days needs attention: rollcall doctor, +// GET /api/v1/ca/intermediates, the web UI's overview, and the rotation +// status for one from the old root. + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "log" + "math" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "text/tabwriter" + "time" +) + +type Intermediate struct { + Name string `json:"name"` + Permits []string `json:"permits"` + Days int `json:"days"` + NotAfter time.Time `json:"not_after"` + Issuer string `json:"issuer"` // fingerprint of the root that signed it + Created time.Time `json:"created"` + Renewed *time.Time `json:"renewed,omitempty"` + LastSeen *time.Time `json:"last_seen,omitempty"` // the host's last renewal request + LastError string `json:"last_error,omitempty"` + Problem string `json:"problem,omitempty"` + cert string + tokenHash string +} + +const ( + interWarnLeft = 30 * 24 * time.Hour // under this to its end: attention + interSilent = 72 * time.Hour // the host has not asked for this long: attention +) + +// problem: what needs attention about x, "" when nothing does +func (x *Intermediate) problem(activeRoot string, now time.Time) string { + var p []string + if left := x.NotAfter.Sub(now); left < interWarnLeft { + if left <= 0 { + p = append(p, "expired "+x.NotAfter.Format("2006-01-02")) + } else { + p = append(p, fmt.Sprintf("expires in %d days (%s)", int(math.Ceil(left.Hours()/24-1e-9)), x.NotAfter.Format("2006-01-02"))) + } + } + if x.LastError != "" { + p = append(p, "last renewal failed: "+x.LastError) + } + switch { + case x.LastSeen == nil && now.Sub(x.Created) > interSilent: + p = append(p, "its host never asked for renewal: install deploy/rollcall-intermediate-renew there") + case x.LastSeen != nil && now.Sub(*x.LastSeen) > interSilent: + p = append(p, "its host has not asked since "+x.LastSeen.Format("2006-01-02 15:04")) + } + if activeRoot != "" && x.Issuer != activeRoot { + p = append(p, "signed by a root that no longer signs: renewed on its host's next request") + } + return strings.Join(p, "; ") +} + +// due: whether a renewal request gets a new certificate +func (x *Intermediate) due(activeRoot string, now time.Time) bool { + c, err := parseCertPEM([]byte(x.cert)) + if err != nil { + return true + } + life := c.NotAfter.Sub(c.NotBefore) + return c.NotAfter.Sub(now) < life/3 || x.Issuer != activeRoot +} + +// ---------------------------------------------------------------- store + +const interCols = `name, permits, days, cert, not_after, issuer, created, renewed, last_seen, coalesce(last_error,''), token_hash` + +func scanInter(row interface{ Scan(...any) error }) (*Intermediate, error) { + var x Intermediate + var permits string + if err := row.Scan(&x.Name, &permits, &x.Days, &x.cert, &x.NotAfter, &x.Issuer, &x.Created, &x.Renewed, &x.LastSeen, &x.LastError, &x.tokenHash); err != nil { + return nil, err + } + x.Permits = splitList(permits) + return &x, nil +} + +func (s *Store) Intermediates(ctx context.Context) ([]*Intermediate, error) { + rows, err := s.pool.Query(ctx, `SELECT `+interCols+` FROM intermediates ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*Intermediate{} + for rows.Next() { + x, err := scanInter(rows) + if err != nil { + return nil, err + } + out = append(out, x) + } + return out, rows.Err() +} + +func (s *Store) Intermediate(ctx context.Context, name string) (*Intermediate, error) { + x, err := scanInter(s.pool.QueryRow(ctx, `SELECT `+interCols+` FROM intermediates WHERE name=$1`, name)) + if err != nil && strings.Contains(err.Error(), "no rows") { + return nil, errNotFound + } + return x, err +} + +func (s *Store) PutIntermediate(ctx context.Context, x *Intermediate) error { + _, err := s.pool.Exec(ctx, `INSERT INTO intermediates(name, permits, days, cert, not_after, issuer, token_hash) VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (name) DO UPDATE SET permits=$2, days=$3, cert=$4, not_after=$5, issuer=$6, token_hash=$7, created=now(), renewed=NULL, last_seen=NULL, last_error=NULL`, + x.Name, strings.Join(x.Permits, ","), x.Days, x.cert, x.NotAfter, x.Issuer, x.tokenHash) + return err +} + +// RenewedIntermediate stores a renewal (or, with cert "", only that the host asked, and how it went). +func (s *Store) RenewedIntermediate(ctx context.Context, name, cert string, notAfter time.Time, issuer, lastErr string) error { + if cert == "" { + _, err := s.pool.Exec(ctx, `UPDATE intermediates SET last_seen=now(), last_error=nullif($2,'') WHERE name=$1`, name, lastErr) + return err + } + _, err := s.pool.Exec(ctx, `UPDATE intermediates SET cert=$2, not_after=$3, issuer=$4, renewed=now(), last_seen=now(), last_error=NULL WHERE name=$1`, name, cert, notAfter, issuer) + return err +} + +func (s *Store) DeleteIntermediate(ctx context.Context, name string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM intermediates WHERE name=$1`, name) + return err +} + +// ---------------------------------------------------------------- signing + +// csrKey: the public key of a PEM CSR whose signature holds +func csrKey(b []byte) (any, error) { + blk, _ := pem.Decode(b) + if blk == nil || blk.Type != "CERTIFICATE REQUEST" && blk.Type != "NEW CERTIFICATE REQUEST" { + return nil, errors.New("not a PEM certificate request") + } + csr, err := x509.ParseCertificateRequest(blk.Bytes) + if err != nil { + return nil, err + } + if err := csr.CheckSignature(); err != nil { + return nil, fmt.Errorf("CSR signature: %w", err) + } + return csr.PublicKey, nil +} + +func (a *App) signIntermediate(name string, permits []string, days int, csrPEM []byte) (string, *x509.Certificate, error) { + pub, err := csrKey(csrPEM) + if err != nil { + return "", nil, err + } + dns, nets, err := parsePermits(strings.Join(permits, ",")) + if err != nil { + return "", nil, err + } + certPEM, cert, err := a.ca.intermediate(pub, name, dns, nets, time.Duration(days)*24*time.Hour) + return string(certPEM), cert, err +} + +var interNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`) + +// splitList: the items of a comma-separated list, trimmed, without empty ones +func splitList(s string) []string { + var out []string + for _, x := range strings.Split(s, ",") { + if x = strings.TrimSpace(x); x != "" { + out = append(out, x) + } + } + return out +} + +func hashToken(t string) string { s := sha256.Sum256([]byte(t)); return hex.EncodeToString(s[:]) } + +// ---------------------------------------------------------------- handlers + +// handleIntermediates: GET lists them with what needs attention; POST signs +// one ({"name","permits","csr","days","replace"}) and returns its +// certificate, the root and the renewal token (shown this once); DELETE +// ?name= stops keeping it (it stays valid until its end: there is no CRL). +func (a *App) handleIntermediates(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + switch r.Method { + case http.MethodGet: + xs, err := a.store.Intermediates(ctx) + if err != nil { + fail(w, 500, err) + return + } + for _, x := range xs { + x.Problem = x.problem(fpHex(a.ca.Cert), time.Now()) + } + writeJSON(w, 200, xs) + case http.MethodDelete: + if err := a.store.DeleteIntermediate(ctx, r.URL.Query().Get("name")); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, obj{"deleted": r.URL.Query().Get("name")}) + case http.MethodPost: + var req struct { + Name, CSR string + Permits []string + Days int + Replace bool + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil || !interNameRe.MatchString(req.Name) || req.CSR == "" { + fail(w, 400, errors.New("name (letters, digits, - . _) and csr are required")) + return + } + if len(req.Permits) == 0 { + fail(w, 400, errors.New("an intermediate needs permits (DNS domains and CIDRs it may issue for)")) + return + } + if req.Days == 0 { + req.Days = 365 + } + req.Days = max(1, min(req.Days, 1825)) + if _, err := a.store.Intermediate(ctx, req.Name); err == nil && !req.Replace { + fail(w, 409, fmt.Errorf("%s is kept already: replace to sign it again with a new token", req.Name)) + return + } + cert, c, err := a.signIntermediate(req.Name, req.Permits, req.Days, []byte(req.CSR)) + if err != nil { + fail(w, 400, err) + return + } + tok := make([]byte, 32) + _, _ = rand.Read(tok) + token := hex.EncodeToString(tok) + x := &Intermediate{Name: req.Name, Permits: req.Permits, Days: req.Days, NotAfter: c.NotAfter, Issuer: fpHex(a.ca.Cert), cert: cert, tokenHash: hashToken(token)} + if err := a.store.PutIntermediate(ctx, x); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, obj{"name": x.Name, "cert": cert, "root": string(a.ca.PEM), "not_after": x.NotAfter, "token": token}) + } +} + +// handleIntermediateRenew: the intermediate's host, with its token and a CSR +// in the body; a new certificate when one is due (?force=1: now), else +// {"renewed": false}. With Accept: application/x-pem-file the answer is the +// certificate itself, or 204 when nothing is due (for a shell script). HTTPS only. +func (a *App) handleIntermediateRenew(w http.ResponseWriter, r *http.Request) { + ctx, name := r.Context(), r.PathValue("name") + if r.TLS == nil && a.cfg.AdminHTTP != "allow" { + host, _, _ := net.SplitHostPort(r.RemoteAddr) + if ip := net.ParseIP(host); a.cfg.AdminHTTP != "loopback" || ip == nil || !ip.IsLoopback() { + fail(w, 403, errors.New("renewal is HTTPS-only")) + return + } + } + x, err := a.store.Intermediate(ctx, name) + tok, _ := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if err != nil || !tokenEqual(hashToken(tok), x.tokenHash) { + fail(w, 401, errors.New("unknown intermediate or wrong token")) // one answer for both: names are not probed + return + } + csr, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<16)) + if err != nil { + fail(w, 400, err) + return + } + active := fpHex(a.ca.Cert) + asPEM := strings.Contains(r.Header.Get("Accept"), "application/x-pem-file") + if !x.due(active, time.Now()) && r.URL.Query().Get("force") != "1" { + _ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", "") + if asPEM { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, 200, obj{"renewed": false, "not_after": x.NotAfter}) + return + } + cert, c, err := a.signIntermediate(x.Name, x.Permits, x.Days, csr) // the constraints it was given, never others + if err != nil { + _ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", err.Error()) + fail(w, 400, err) + return + } + if err := a.store.RenewedIntermediate(ctx, name, cert, c.NotAfter, active, ""); err != nil { + fail(w, 500, err) + return + } + log.Printf("ca: intermediate %s renewed until %s", name, c.NotAfter.Format("2006-01-02")) + if asPEM { + w.Header().Set("Content-Type", "application/x-pem-file") + _, _ = io.WriteString(w, cert) + return + } + writeJSON(w, 200, obj{"renewed": true, "cert": cert, "root": string(a.ca.PEM), "not_after": c.NotAfter}) +} + +// ---------------------------------------------------------------- CLI + +// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR] [-replace] +func caIntermediateCmd(name string, args []string) error { + fs := newFlags("ca intermediate") + permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)") + csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated here") + days := fs.Int("days", 365, "validity in days") + out := fs.String("out", ".", "directory for NAME.pem, NAME.token (+ NAME-key.pem without -csr)") + replace := fs.Bool("replace", false, "sign again an intermediate the region keeps already (a new renewal token)") + _ = fs.Parse(args) + permits := splitList(*permit) + if len(permits) == 0 { + return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16)") + } + if _, _, err := parsePermits(*permit); err != nil { + return err + } + var csrPEM, keyPEM []byte + var err error + if *csrFile != "" { + if csrPEM, err = os.ReadFile(*csrFile); err != nil { + return err + } + } else { + var csr string + if keyPEM, csr, err = newCSR(name); err != nil { + return err + } + csrPEM = []byte(csr) + } + c, err := newClient() + if err != nil { + return err + } + var res struct { + Cert, Root, Token string + NotAfter time.Time `json:"not_after"` + } + if err := c.do("POST", "/api/v1/ca/intermediates", obj{"name": name, "permits": permits, "csr": string(csrPEM), "days": *days, "replace": *replace}, &res); err != nil { + return err + } + write := func(f string, b []byte, mode os.FileMode) error { return os.WriteFile(filepath.Join(*out, f), b, mode) } + if keyPEM != nil { + if err := write(name+"-key.pem", keyPEM, 0o600); err != nil { + return err + } + } + if err := write(name+".pem", []byte(res.Cert), 0o644); err != nil { + return err + } + if err := write(name+".token", []byte(res.Token+"\n"), 0o600); err != nil { + return err + } + fmt.Printf("wrote %s.pem and %s.token in %s: intermediate CA %q until %s, may issue for %s\n", name, name, *out, name, res.NotAfter.Format("2006-01-02"), strings.Join(permits, " ")) + fmt.Printf("the region renews it: on the host with its key, install deploy/rollcall-intermediate-renew with\n URL=%s NAME=%s TOKEN_FILE=<%s.token there> CERT= KEY=\n", c.base, name, name) + return nil +} + +// rollcall ca intermediates: the intermediates the region keeps +func caIntermediatesCmd() error { + c, err := newClient() + if err != nil { + return err + } + var xs []Intermediate + if err := c.do("GET", "/api/v1/ca/intermediates", nil, &xs); err != nil { + return err + } + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tUNTIL\tLAST ASKED\tPERMITS\tATTENTION") + for _, x := range xs { + seen := "never" + if x.LastSeen != nil { + seen = x.LastSeen.Format("2006-01-02 15:04") + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", x.Name, x.NotAfter.Format("2006-01-02"), seen, strings.Join(x.Permits, " "), orDash(x.Problem)) + } + return tw.Flush() +} diff --git a/cmd/rollcall/intermediates_test.go b/cmd/rollcall/intermediates_test.go new file mode 100644 index 0000000..6a9706a --- /dev/null +++ b/cmd/rollcall/intermediates_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// a renewal signs with the constraints the region keeps, whatever the CSR asks for +func TestIntermediateRenewalKeepsConstraints(t *testing.T) { + ca, err := loadOrCreateCA(t.TempDir(), "test CA") + if err != nil { + t.Fatal(err) + } + a := &App{ca: ca} + _, csr, err := newCSR("evil.example.org") // the name in the CSR does not matter + if err != nil { + t.Fatal(err) + } + pemCert, c, err := a.signIntermediate("caddy", []string{"int.example.com", "10.213.0.0/16"}, 90, []byte(csr)) + if err != nil { + t.Fatal(err) + } + if !c.IsCA || c.Subject.CommonName != "caddy" || len(c.PermittedDNSDomains) != 1 || c.PermittedDNSDomains[0] != "int.example.com" || + len(c.PermittedIPRanges) != 1 || c.PermittedIPRanges[0].String() != "10.213.0.0/16" || !c.PermittedDNSDomainsCritical { + t.Fatalf("constraints: %+v %v %v", c.Subject, c.PermittedDNSDomains, c.PermittedIPRanges) + } + if d := c.NotAfter.Sub(c.NotBefore); d < 89*24*time.Hour || d > 91*24*time.Hour { + t.Fatalf("validity %v", d) + } + if _, _, err := a.signIntermediate("caddy", []string{"int.example.com"}, 90, []byte("not a csr")); err == nil { + t.Fatal("signed something that is not a CSR") + } + + // due in the last third of its life, or when another root signs now + x := &Intermediate{Name: "caddy", cert: pemCert, Issuer: fpHex(ca.Cert)} + if x.due(fpHex(ca.Cert), time.Now()) { + t.Fatal("due right after signing") + } + if !x.due(fpHex(ca.Cert), time.Now().Add(61*24*time.Hour)) { + t.Fatal("not due with a third of its life left") + } + if !x.due("another-root", time.Now()) { + t.Fatal("not due after a rotation") + } +} + +func TestIntermediateProblems(t *testing.T) { + now := time.Now() + ago := func(d time.Duration) *time.Time { x := now.Add(-d); return &x } + for _, c := range []struct { + x Intermediate + want string // a fragment, "" for none + }{ + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-time.Hour)}, ""}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour), LastSeen: ago(time.Hour)}, ""}, + {Intermediate{NotAfter: now.Add(10 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expires in 10 days"}, + {Intermediate{NotAfter: now.Add(-time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expired"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour)}, "never asked"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(100 * time.Hour)}, "has not asked since"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour), LastError: "CSR signature"}, "last renewal failed"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "old", LastSeen: ago(time.Hour)}, "no longer signs"}, + } { + got := c.x.problem("r", now) + if (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v: %q, want %q", c.x, got, c.want) + } + } + if hashToken("a") == hashToken("b") || len(hashToken("a")) != 64 { + t.Fatal("hashToken") + } + if got := splitList(" a, ,b ,"); len(got) != 2 || got[0] != "a" || got[1] != "b" { + t.Fatalf("splitList %q", got) + } +} diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 63c92f9..6623146 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -32,7 +32,8 @@ func usage() { rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) - rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA + rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA (the region renews it) + rollcall ca intermediates the intermediates the region keeps, and what needs attention rollcall ca rotate [status | prepare | switch | retire] [-force] replace the root without re-joining racks rollcall version rollcall keygen diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 11bac09..f0b743a 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -252,6 +252,10 @@ func (a *App) routes() http.Handler { mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete)) mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM)) mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation)) + mux.HandleFunc("GET /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("POST /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("DELETE /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("POST /api/v1/ca/intermediates/{name}/renew", a.handleIntermediateRenew) // its own token (intermediates.go) mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS)) mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync)) mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts)) diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index a0b0ea3..ad2306f 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -115,6 +115,19 @@ CREATE TABLE IF NOT EXISTS switches ( source text NOT NULL DEFAULT 'local' ); ALTER TABLE switches ADD COLUMN IF NOT EXISTS role text; +CREATE TABLE IF NOT EXISTS intermediates ( + name text PRIMARY KEY, + permits text NOT NULL, + days int NOT NULL, + cert text NOT NULL, + not_after timestamptz NOT NULL, + issuer text NOT NULL, + token_hash text NOT NULL, + created timestamptz NOT NULL DEFAULT now(), + renewed timestamptz, + last_seen timestamptz, + last_error text +); CREATE TABLE IF NOT EXISTS cables ( machine_id text NOT NULL REFERENCES machines(id) ON DELETE CASCADE, port text NOT NULL, diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index be3f5b5..3f79b81 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -91,7 +91,7 @@ async function refresh(full) { $('#n-net').textContent = off ? `${off} offline` : (subnets.length || ''); const page = route().page; if (full || page === 'overview' || page === 'jobs') S.jobs = await api('GET', '/api/v1/jobs'); - if (full || page === 'overview') S.events = await api('GET', '/api/v1/events'); + if (full || page === 'overview') [S.events, S.inter] = await Promise.all([api('GET', '/api/v1/events'), api('GET', '/api/v1/ca/intermediates').catch(() => [])]); if (full || page === 'sites') [S.sites, S.rooms, S.cabling] = await Promise.all([api('GET', '/api/v1/sites'), api('GET', '/api/v1/rooms'), api('GET', '/api/v1/cabling').catch(() => null)]); if (full || page === 'overview' || page === 'network') { [S.ips, S.ipam] = await Promise.all([api('GET', '/api/v1/ips'), api('GET', '/api/v1/ipam').catch(() => null)]); @@ -181,7 +181,10 @@ function pageOverview() { ['Jobs failed (last 100)', failed.length ? `${failed.length}` : '0'], ['Oldest ready', (() => { const r = ready.slice().sort((a, b) => new Date(a.last_seen) - new Date(b.last_seen))[0]; return r ? `${esc(nameOf(r))} · ${ago(r.last_seen)}` : '—'; })()]])}`; // attention + const inter = S.inter || [], interBad = inter.filter(x => x.problem); const attention = kv([ + ...(inter.length ? [['Intermediate CAs', interBad.length ? interBad.map(x => `${esc(x.name)}: ${esc(x.problem)}`).join('
') + : `${inter.length} renewed by their hosts (next end ${esc(inter.map(x => x.not_after).sort()[0].slice(0, 10))})`]] : []), ['Rack controllers', offline.length ? `${offline.map(c => esc(c.name)).join(', ')} offline show` : `${S.controllers.length} online`], ['External IPAM', !S.ipam || !S.ipam.provider ? 'built-in only' : ipamLast && ipamLast.error ? `${esc(ipamLast.error.slice(0, 80))}` : `${esc(S.ipam.provider)} in sync${ipamLast && ipamLast.conflicts && ipamLast.conflicts.length ? ` ${ipamLast.conflicts.length} conflicts` : ''} ${ipamLast && ipamLast.at ? ago(ipamLast.at) + ' ago' : ''}`], ['BMC not configured', bmcBad ? `${bmcBad} show` : 'none'], diff --git a/deploy/rollcall-intermediate-renew.sh b/deploy/rollcall-intermediate-renew.sh new file mode 100644 index 0000000..c17a5a2 --- /dev/null +++ b/deploy/rollcall-intermediate-renew.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# rollcall-intermediate-renew [-force] NAME: keep a delegated intermediate CA +# current. Runs daily on the host that holds the intermediate's key (an ACME +# server: Caddy, step-ca), from /etc/rollcall-intermediate/NAME.env: +# URL=https://region.example the region (HTTPS) +# TOKEN_FILE=/etc/rollcall-intermediate/NAME.token from `rollcall ca intermediate` +# CERT=/etc/caddy/pki/int.pem the intermediate's certificate (replaced) +# KEY=/etc/caddy/pki/int-key.pem its key (stays; the CSR is made from it) +# ROOT=/etc/caddy/pki/root.pem optional: kept to the active root +# CA=/etc/rollcall-intermediate/NAME-bundle.pem the region's roots (seeded from ROOT, kept current) +# RELOAD='systemctl restart caddy' optional: after a new certificate +# The region decides whether a new certificate is due (the last third of its +# life, or a root rotation) and signs with the constraints it was given. +set -eu +force= +[ "${1:-}" = -force ] && { force=1; shift; } +NAME=${1:?usage: rollcall-intermediate-renew [-force] NAME} +. "/etc/rollcall-intermediate/$NAME.env" +: "${URL:?}" "${TOKEN_FILE:?}" "${CERT:?}" "${KEY:?}" +CA=${CA:-/etc/rollcall-intermediate/$NAME-bundle.pem} +if [ ! -s "$CA" ]; then + [ -n "${ROOT:-}" ] && [ -s "$ROOT" ] || { echo "rollcall: $CA missing and no ROOT to start it from" >&2; exit 1; } + cp "$ROOT" "$CA" +fi +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT + +# the region's roots, checked against the ones held: a rotation reaches here during prepare +if curl -fsS --max-time 30 --cacert "$CA" -o "$T/bundle" "$URL/ca-bundle.pem" && grep -q 'BEGIN CERTIFICATE' "$T/bundle"; then + cmp -s "$T/bundle" "$CA" || { cat "$T/bundle" > "$CA"; echo "rollcall: $NAME: the region's roots changed; $CA updated"; } +fi + +openssl req -new -key "$KEY" -subj "/CN=$NAME" -out "$T/csr" 2>/dev/null +q=; [ -n "$force" ] && q='?force=1' +code=$(curl -sS --max-time 60 --cacert "$CA" -o "$T/cert" -w '%{http_code}' -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ + -H 'Accept: application/x-pem-file' --data-binary @"$T/csr" "$URL/api/v1/ca/intermediates/$NAME/renew$q") || { echo "rollcall: $NAME: the region did not answer" >&2; exit 1; } +case $code in + 204) exit 0 ;; # nothing due + 200) ;; + *) echo "rollcall: $NAME: renewal refused (HTTP $code): $(cat "$T/cert")" >&2; exit 1 ;; +esac + +# what came back: a CA certificate for this key, from a root held here +openssl x509 -in "$T/cert" -noout -text | grep -q 'CA:TRUE' || { echo "rollcall: $NAME: the answer is not a CA certificate" >&2; exit 1; } +[ "$(openssl x509 -in "$T/cert" -noout -pubkey)" = "$(openssl pkey -in "$KEY" -pubout)" ] || { echo "rollcall: $NAME: the certificate is not for $KEY" >&2; exit 1; } +openssl verify -CAfile "$CA" "$T/cert" >/dev/null || { echo "rollcall: $NAME: the certificate does not verify against $CA" >&2; exit 1; } +put() { # $1 new content file, $2 target: same owner and mode, the old one kept as .prev + [ -f "$2" ] && cp -p "$2" "$2.prev" + if [ -f "$2" ]; then cat "$1" > "$2"; else install -m 0644 "$1" "$2"; fi +} +put "$T/cert" "$CERT" +if [ -n "${ROOT:-}" ]; then # the active root: the first certificate of the bundle + awk '/-----BEGIN CERTIFICATE-----/{n++} n==1{print} /-----END CERTIFICATE-----/ && n==1{exit}' "$CA" > "$T/root" + cmp -s "$T/root" "$ROOT" || put "$T/root" "$ROOT" +fi +echo "rollcall: $NAME renewed: $(openssl x509 -in "$CERT" -noout -enddate)" +[ -n "${RELOAD:-}" ] && sh -c "$RELOAD" +exit 0 diff --git a/deploy/rollcall-intermediate-renew@.service b/deploy/rollcall-intermediate-renew@.service new file mode 100644 index 0000000..0f946dc --- /dev/null +++ b/deploy/rollcall-intermediate-renew@.service @@ -0,0 +1,8 @@ +[Unit] +Description=rollcall: renew the delegated intermediate CA %i +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/rollcall-intermediate-renew %i diff --git a/deploy/rollcall-intermediate-renew@.timer b/deploy/rollcall-intermediate-renew@.timer new file mode 100644 index 0000000..aefd053 --- /dev/null +++ b/deploy/rollcall-intermediate-renew@.timer @@ -0,0 +1,11 @@ +[Unit] +Description=rollcall: renew the delegated intermediate CA %i daily + +[Timer] +OnBootSec=15min +OnUnitActiveSec=1d +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target