diff --git a/CHANGELOG.md b/CHANGELOG.md index c976e79..2475fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ publish a tag that has no section here. ## [Unreleased] +### Added +- Intermediates renew themselves: the region keeps the intermediates it signs (`rollcall ca + intermediate` goes through the API now and prints a renewal token once; `rollcall ca + intermediates` lists them), and `deploy/rollcall-intermediate-renew.sh` with its daily timer on + the intermediate's host gets a new certificate in the last third of its life or after a root + rotation, always with the first name constraints. An intermediate near its end, failing to renew + or whose host went silent shows in `rollcall doctor`, on the overview and in the rotation status. + (#14) + ## [0.4.0] - 2026-11-01 ### Added diff --git a/README.md b/README.md index 5f67969..72d6c84 100644 --- a/README.md +++ b/README.md @@ -587,6 +587,34 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and can be issued. Roots created before this (path length 0) cannot delegate: rotate the root (below). `deploy/tags/acme-cert.yaml` is a tag that fetches a node certificate at first boot and renews it. +* **Intermediates renew themselves.** The region keeps every intermediate it + signs and prints a renewal token once (`NAME.token` next to `NAME.pem`). On + the host with the intermediate's key, `deploy/rollcall-intermediate-renew.sh` + (installed as `/usr/local/sbin/rollcall-intermediate-renew`) and the + `rollcall-intermediate-renew@NAME.timer` ask the region every day with that + token and a CSR from the same key; settings in + `/etc/rollcall-intermediate/NAME.env`: + + ``` + URL=https://region.example + TOKEN_FILE=/etc/rollcall-intermediate/caddy.token + CERT=/etc/caddy/pki/int.pem + KEY=/etc/caddy/pki/int-key.pem + ROOT=/etc/caddy/pki/root.pem # optional: kept to the active root + RELOAD='systemctl restart caddy' # optional: after a new certificate + ``` + + The region signs again in the last third of the certificate's life, or right + away when another root signs since a rotation's `switch`, always with the + name constraints it was first given; the script checks what comes back (a CA + certificate for its key, from a root it holds), keeps the old file as + `.prev`, keeps the region's roots current from `/ca-bundle.pem`, and runs + `RELOAD`. `rollcall ca intermediates` lists them; one under 30 days from its + end, whose renewal failed or whose host has not asked for three days shows in + `rollcall doctor`, on the web UI's overview and in the rotation status. + `-replace` signs one the region keeps again with a new token (a lost token); + `DELETE /api/v1/ca/intermediates?name=` stops keeping it (it stays valid to + its end: there is no revocation list). * **Root rotation without re-joining.** In three steps, each followed by a restart of the region (`systemctl restart rollcall`): diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index 78751cc..cc8c86f 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -684,7 +684,9 @@ func caCmd(args []string) error { } switch { case args[0] == "intermediate" && len(args) >= 2: - return caIntermediateCmd(cfg, args[1], args[2:]) + return caIntermediateCmd(args[1], args[2:]) + case args[0] == "intermediates": + return caIntermediatesCmd() case args[0] == "rotate": return caRotateCmd(cfg, args[1:]) case args[0] != "issue" || len(args) < 2: @@ -723,76 +725,6 @@ func caCmd(args []string) error { return nil } -// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR] -func caIntermediateCmd(cfg *Config, name string, args []string) error { - fs := newFlags("ca intermediate") - permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)") - csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated") - days := fs.Int("days", 365, "validity in days") - out := fs.String("out", ".", "directory for NAME.pem (+ NAME-key.pem without -csr)") - unconstrained := fs.Bool("unconstrained", false, "allow an intermediate without name constraints") - _ = fs.Parse(args) - dns, nets, err := parsePermits(*permit) - if err != nil { - return err - } - if len(dns)+len(nets) == 0 && !*unconstrained { - return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16); -unconstrained to override") - } - ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - return err - } - var pub crypto.PublicKey - var keyPEM []byte - if *csrFile != "" { - b, err := os.ReadFile(*csrFile) - if err != nil { - return err - } - blk, _ := pem.Decode(b) - if blk == nil { - return fmt.Errorf("%s: not PEM", *csrFile) - } - csr, err := x509.ParseCertificateRequest(blk.Bytes) - if err != nil { - return err - } - if err := csr.CheckSignature(); err != nil { - return fmt.Errorf("CSR signature: %w", err) - } - pub = csr.PublicKey - } else { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return err - } - pub = &key.PublicKey - kder, _ := x509.MarshalECPrivateKey(key) - keyPEM = pemEncode("EC PRIVATE KEY", kder) - } - certPEM, cert, err := ca.intermediate(pub, name, dns, nets, time.Duration(*days)*24*time.Hour) - if err != nil { - return err - } - if keyPEM != nil { - if err := os.WriteFile(filepath.Join(*out, name+"-key.pem"), keyPEM, 0o600); err != nil { - return err - } - } - if err := os.WriteFile(filepath.Join(*out, name+".pem"), certPEM, 0o644); err != nil { - return err - } - var permits []string - permits = append(permits, dns...) - for _, n := range nets { - permits = append(permits, n.String()) - } - fmt.Printf("wrote %s.pem in %s: intermediate CA %q until %s, may issue for %s\nroot: %s (%s)\n", name, *out, name, - cert.NotAfter.Format("2006-01-02"), orDash(strings.Join(permits, " ")), filepath.Join(cfg.CADir, "ca.pem"), ca.Fingerprint()) - return nil -} - // fileOwner returns uid, gid of path (unix). func fileOwner(path string) ([2]int, bool) { st, err := os.Stat(path) diff --git a/cmd/rollcall/carot.go b/cmd/rollcall/carot.go index 6c7d199..f1e99bc 100644 --- a/cmd/rollcall/carot.go +++ b/cmd/rollcall/carot.go @@ -234,6 +234,13 @@ func (a *App) rotation(ctx context.Context) (rotationStatus, error) { } } } + if xs, err := a.store.Intermediates(ctx); err == nil { // their hosts renew them from the new root on their own + for _, x := range xs { + if x.Issuer != st.Active { + st.Warnings = append(st.Warnings, fmt.Sprintf("intermediate %s is from the old root: its host renews it on its next daily request (or now: rollcall-intermediate-renew -force there)", x.Name)) + } + } + } racks, err := a.store.RackTrust(ctx) if err != nil { return st, err diff --git a/cmd/rollcall/doctor.go b/cmd/rollcall/doctor.go index 43d8f75..c1905ec 100644 --- a/cmd/rollcall/doctor.go +++ b/cmd/rollcall/doctor.go @@ -83,6 +83,24 @@ func doctorCmd(args []string) error { d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)") } else { d.ok("database", "connected, schema current") + // delegated intermediates the region renews: ending, failing, or a silent host + active := "" + if b, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + active = fpHex(c) + } + } + ictx, icancel := context.WithTimeout(context.Background(), 10*time.Second) + if xs, err := st.Intermediates(ictx); err == nil { + for _, x := range xs { + if p := x.problem(active, time.Now()); p != "" { + d.warn("intermediate "+x.Name, p, "on its host: systemctl status rollcall-intermediate-renew; rollcall ca intermediates") + } else { + d.ok("intermediate "+x.Name, "until "+x.NotAfter.Format("2006-01-02")+", renewed by its host") + } + } + } + icancel() st.pool.Close() } diff --git a/cmd/rollcall/intermediates.go b/cmd/rollcall/intermediates.go new file mode 100644 index 0000000..872094d --- /dev/null +++ b/cmd/rollcall/intermediates.go @@ -0,0 +1,401 @@ +package main + +// Delegated intermediates kept current. The region keeps every intermediate +// it signs (its name constraints, certificate and a renewal token shown +// once), and the host that holds the intermediate's key asks every day with +// that token and a CSR (POST /api/v1/ca/intermediates/NAME/renew; the +// deploy/rollcall-intermediate-renew script and timer). The region signs +// again when the certificate is in the last third of its life or comes from +// a root that is no longer the active one (after a rotation's switch), +// always with the constraints it was first given: a renewal never widens +// what the intermediate may issue. Otherwise it answers that nothing is due. +// +// An intermediate under 30 days from its end, whose last renewal failed, or +// whose host has not asked for three days needs attention: rollcall doctor, +// GET /api/v1/ca/intermediates, the web UI's overview, and the rotation +// status for one from the old root. + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "log" + "math" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "text/tabwriter" + "time" +) + +type Intermediate struct { + Name string `json:"name"` + Permits []string `json:"permits"` + Days int `json:"days"` + NotAfter time.Time `json:"not_after"` + Issuer string `json:"issuer"` // fingerprint of the root that signed it + Created time.Time `json:"created"` + Renewed *time.Time `json:"renewed,omitempty"` + LastSeen *time.Time `json:"last_seen,omitempty"` // the host's last renewal request + LastError string `json:"last_error,omitempty"` + Problem string `json:"problem,omitempty"` + cert string + tokenHash string +} + +const ( + interWarnLeft = 30 * 24 * time.Hour // under this to its end: attention + interSilent = 72 * time.Hour // the host has not asked for this long: attention +) + +// problem: what needs attention about x, "" when nothing does +func (x *Intermediate) problem(activeRoot string, now time.Time) string { + var p []string + if left := x.NotAfter.Sub(now); left < interWarnLeft { + if left <= 0 { + p = append(p, "expired "+x.NotAfter.Format("2006-01-02")) + } else { + p = append(p, fmt.Sprintf("expires in %d days (%s)", int(math.Ceil(left.Hours()/24-1e-9)), x.NotAfter.Format("2006-01-02"))) + } + } + if x.LastError != "" { + p = append(p, "last renewal failed: "+x.LastError) + } + switch { + case x.LastSeen == nil && now.Sub(x.Created) > interSilent: + p = append(p, "its host never asked for renewal: install deploy/rollcall-intermediate-renew there") + case x.LastSeen != nil && now.Sub(*x.LastSeen) > interSilent: + p = append(p, "its host has not asked since "+x.LastSeen.Format("2006-01-02 15:04")) + } + if activeRoot != "" && x.Issuer != activeRoot { + p = append(p, "signed by a root that no longer signs: renewed on its host's next request") + } + return strings.Join(p, "; ") +} + +// due: whether a renewal request gets a new certificate +func (x *Intermediate) due(activeRoot string, now time.Time) bool { + c, err := parseCertPEM([]byte(x.cert)) + if err != nil { + return true + } + life := c.NotAfter.Sub(c.NotBefore) + return c.NotAfter.Sub(now) < life/3 || x.Issuer != activeRoot +} + +// ---------------------------------------------------------------- store + +const interCols = `name, permits, days, cert, not_after, issuer, created, renewed, last_seen, coalesce(last_error,''), token_hash` + +func scanInter(row interface{ Scan(...any) error }) (*Intermediate, error) { + var x Intermediate + var permits string + if err := row.Scan(&x.Name, &permits, &x.Days, &x.cert, &x.NotAfter, &x.Issuer, &x.Created, &x.Renewed, &x.LastSeen, &x.LastError, &x.tokenHash); err != nil { + return nil, err + } + x.Permits = splitList(permits) + return &x, nil +} + +func (s *Store) Intermediates(ctx context.Context) ([]*Intermediate, error) { + rows, err := s.pool.Query(ctx, `SELECT `+interCols+` FROM intermediates ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []*Intermediate{} + for rows.Next() { + x, err := scanInter(rows) + if err != nil { + return nil, err + } + out = append(out, x) + } + return out, rows.Err() +} + +func (s *Store) Intermediate(ctx context.Context, name string) (*Intermediate, error) { + x, err := scanInter(s.pool.QueryRow(ctx, `SELECT `+interCols+` FROM intermediates WHERE name=$1`, name)) + if err != nil && strings.Contains(err.Error(), "no rows") { + return nil, errNotFound + } + return x, err +} + +func (s *Store) PutIntermediate(ctx context.Context, x *Intermediate) error { + _, err := s.pool.Exec(ctx, `INSERT INTO intermediates(name, permits, days, cert, not_after, issuer, token_hash) VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (name) DO UPDATE SET permits=$2, days=$3, cert=$4, not_after=$5, issuer=$6, token_hash=$7, created=now(), renewed=NULL, last_seen=NULL, last_error=NULL`, + x.Name, strings.Join(x.Permits, ","), x.Days, x.cert, x.NotAfter, x.Issuer, x.tokenHash) + return err +} + +// RenewedIntermediate stores a renewal (or, with cert "", only that the host asked, and how it went). +func (s *Store) RenewedIntermediate(ctx context.Context, name, cert string, notAfter time.Time, issuer, lastErr string) error { + if cert == "" { + _, err := s.pool.Exec(ctx, `UPDATE intermediates SET last_seen=now(), last_error=nullif($2,'') WHERE name=$1`, name, lastErr) + return err + } + _, err := s.pool.Exec(ctx, `UPDATE intermediates SET cert=$2, not_after=$3, issuer=$4, renewed=now(), last_seen=now(), last_error=NULL WHERE name=$1`, name, cert, notAfter, issuer) + return err +} + +func (s *Store) DeleteIntermediate(ctx context.Context, name string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM intermediates WHERE name=$1`, name) + return err +} + +// ---------------------------------------------------------------- signing + +// csrKey: the public key of a PEM CSR whose signature holds +func csrKey(b []byte) (any, error) { + blk, _ := pem.Decode(b) + if blk == nil || blk.Type != "CERTIFICATE REQUEST" && blk.Type != "NEW CERTIFICATE REQUEST" { + return nil, errors.New("not a PEM certificate request") + } + csr, err := x509.ParseCertificateRequest(blk.Bytes) + if err != nil { + return nil, err + } + if err := csr.CheckSignature(); err != nil { + return nil, fmt.Errorf("CSR signature: %w", err) + } + return csr.PublicKey, nil +} + +func (a *App) signIntermediate(name string, permits []string, days int, csrPEM []byte) (string, *x509.Certificate, error) { + pub, err := csrKey(csrPEM) + if err != nil { + return "", nil, err + } + dns, nets, err := parsePermits(strings.Join(permits, ",")) + if err != nil { + return "", nil, err + } + certPEM, cert, err := a.ca.intermediate(pub, name, dns, nets, time.Duration(days)*24*time.Hour) + return string(certPEM), cert, err +} + +var interNameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$`) + +// splitList: the items of a comma-separated list, trimmed, without empty ones +func splitList(s string) []string { + var out []string + for _, x := range strings.Split(s, ",") { + if x = strings.TrimSpace(x); x != "" { + out = append(out, x) + } + } + return out +} + +func hashToken(t string) string { s := sha256.Sum256([]byte(t)); return hex.EncodeToString(s[:]) } + +// ---------------------------------------------------------------- handlers + +// handleIntermediates: GET lists them with what needs attention; POST signs +// one ({"name","permits","csr","days","replace"}) and returns its +// certificate, the root and the renewal token (shown this once); DELETE +// ?name= stops keeping it (it stays valid until its end: there is no CRL). +func (a *App) handleIntermediates(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + switch r.Method { + case http.MethodGet: + xs, err := a.store.Intermediates(ctx) + if err != nil { + fail(w, 500, err) + return + } + for _, x := range xs { + x.Problem = x.problem(fpHex(a.ca.Cert), time.Now()) + } + writeJSON(w, 200, xs) + case http.MethodDelete: + if err := a.store.DeleteIntermediate(ctx, r.URL.Query().Get("name")); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, obj{"deleted": r.URL.Query().Get("name")}) + case http.MethodPost: + var req struct { + Name, CSR string + Permits []string + Days int + Replace bool + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil || !interNameRe.MatchString(req.Name) || req.CSR == "" { + fail(w, 400, errors.New("name (letters, digits, - . _) and csr are required")) + return + } + if len(req.Permits) == 0 { + fail(w, 400, errors.New("an intermediate needs permits (DNS domains and CIDRs it may issue for)")) + return + } + if req.Days == 0 { + req.Days = 365 + } + req.Days = max(1, min(req.Days, 1825)) + if _, err := a.store.Intermediate(ctx, req.Name); err == nil && !req.Replace { + fail(w, 409, fmt.Errorf("%s is kept already: replace to sign it again with a new token", req.Name)) + return + } + cert, c, err := a.signIntermediate(req.Name, req.Permits, req.Days, []byte(req.CSR)) + if err != nil { + fail(w, 400, err) + return + } + tok := make([]byte, 32) + _, _ = rand.Read(tok) + token := hex.EncodeToString(tok) + x := &Intermediate{Name: req.Name, Permits: req.Permits, Days: req.Days, NotAfter: c.NotAfter, Issuer: fpHex(a.ca.Cert), cert: cert, tokenHash: hashToken(token)} + if err := a.store.PutIntermediate(ctx, x); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, obj{"name": x.Name, "cert": cert, "root": string(a.ca.PEM), "not_after": x.NotAfter, "token": token}) + } +} + +// handleIntermediateRenew: the intermediate's host, with its token and a CSR +// in the body; a new certificate when one is due (?force=1: now), else +// {"renewed": false}. With Accept: application/x-pem-file the answer is the +// certificate itself, or 204 when nothing is due (for a shell script). HTTPS only. +func (a *App) handleIntermediateRenew(w http.ResponseWriter, r *http.Request) { + ctx, name := r.Context(), r.PathValue("name") + if r.TLS == nil && a.cfg.AdminHTTP != "allow" { + host, _, _ := net.SplitHostPort(r.RemoteAddr) + if ip := net.ParseIP(host); a.cfg.AdminHTTP != "loopback" || ip == nil || !ip.IsLoopback() { + fail(w, 403, errors.New("renewal is HTTPS-only")) + return + } + } + x, err := a.store.Intermediate(ctx, name) + tok, _ := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if err != nil || !tokenEqual(hashToken(tok), x.tokenHash) { + fail(w, 401, errors.New("unknown intermediate or wrong token")) // one answer for both: names are not probed + return + } + csr, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<16)) + if err != nil { + fail(w, 400, err) + return + } + active := fpHex(a.ca.Cert) + asPEM := strings.Contains(r.Header.Get("Accept"), "application/x-pem-file") + if !x.due(active, time.Now()) && r.URL.Query().Get("force") != "1" { + _ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", "") + if asPEM { + w.WriteHeader(http.StatusNoContent) + return + } + writeJSON(w, 200, obj{"renewed": false, "not_after": x.NotAfter}) + return + } + cert, c, err := a.signIntermediate(x.Name, x.Permits, x.Days, csr) // the constraints it was given, never others + if err != nil { + _ = a.store.RenewedIntermediate(ctx, name, "", time.Time{}, "", err.Error()) + fail(w, 400, err) + return + } + if err := a.store.RenewedIntermediate(ctx, name, cert, c.NotAfter, active, ""); err != nil { + fail(w, 500, err) + return + } + log.Printf("ca: intermediate %s renewed until %s", name, c.NotAfter.Format("2006-01-02")) + if asPEM { + w.Header().Set("Content-Type", "application/x-pem-file") + _, _ = io.WriteString(w, cert) + return + } + writeJSON(w, 200, obj{"renewed": true, "cert": cert, "root": string(a.ca.PEM), "not_after": c.NotAfter}) +} + +// ---------------------------------------------------------------- CLI + +// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR] [-replace] +func caIntermediateCmd(name string, args []string) error { + fs := newFlags("ca intermediate") + permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)") + csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated here") + days := fs.Int("days", 365, "validity in days") + out := fs.String("out", ".", "directory for NAME.pem, NAME.token (+ NAME-key.pem without -csr)") + replace := fs.Bool("replace", false, "sign again an intermediate the region keeps already (a new renewal token)") + _ = fs.Parse(args) + permits := splitList(*permit) + if len(permits) == 0 { + return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16)") + } + if _, _, err := parsePermits(*permit); err != nil { + return err + } + var csrPEM, keyPEM []byte + var err error + if *csrFile != "" { + if csrPEM, err = os.ReadFile(*csrFile); err != nil { + return err + } + } else { + var csr string + if keyPEM, csr, err = newCSR(name); err != nil { + return err + } + csrPEM = []byte(csr) + } + c, err := newClient() + if err != nil { + return err + } + var res struct { + Cert, Root, Token string + NotAfter time.Time `json:"not_after"` + } + if err := c.do("POST", "/api/v1/ca/intermediates", obj{"name": name, "permits": permits, "csr": string(csrPEM), "days": *days, "replace": *replace}, &res); err != nil { + return err + } + write := func(f string, b []byte, mode os.FileMode) error { return os.WriteFile(filepath.Join(*out, f), b, mode) } + if keyPEM != nil { + if err := write(name+"-key.pem", keyPEM, 0o600); err != nil { + return err + } + } + if err := write(name+".pem", []byte(res.Cert), 0o644); err != nil { + return err + } + if err := write(name+".token", []byte(res.Token+"\n"), 0o600); err != nil { + return err + } + fmt.Printf("wrote %s.pem and %s.token in %s: intermediate CA %q until %s, may issue for %s\n", name, name, *out, name, res.NotAfter.Format("2006-01-02"), strings.Join(permits, " ")) + fmt.Printf("the region renews it: on the host with its key, install deploy/rollcall-intermediate-renew with\n URL=%s NAME=%s TOKEN_FILE=<%s.token there> CERT= KEY=\n", c.base, name, name) + return nil +} + +// rollcall ca intermediates: the intermediates the region keeps +func caIntermediatesCmd() error { + c, err := newClient() + if err != nil { + return err + } + var xs []Intermediate + if err := c.do("GET", "/api/v1/ca/intermediates", nil, &xs); err != nil { + return err + } + tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tUNTIL\tLAST ASKED\tPERMITS\tATTENTION") + for _, x := range xs { + seen := "never" + if x.LastSeen != nil { + seen = x.LastSeen.Format("2006-01-02 15:04") + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", x.Name, x.NotAfter.Format("2006-01-02"), seen, strings.Join(x.Permits, " "), orDash(x.Problem)) + } + return tw.Flush() +} diff --git a/cmd/rollcall/intermediates_test.go b/cmd/rollcall/intermediates_test.go new file mode 100644 index 0000000..6a9706a --- /dev/null +++ b/cmd/rollcall/intermediates_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// a renewal signs with the constraints the region keeps, whatever the CSR asks for +func TestIntermediateRenewalKeepsConstraints(t *testing.T) { + ca, err := loadOrCreateCA(t.TempDir(), "test CA") + if err != nil { + t.Fatal(err) + } + a := &App{ca: ca} + _, csr, err := newCSR("evil.example.org") // the name in the CSR does not matter + if err != nil { + t.Fatal(err) + } + pemCert, c, err := a.signIntermediate("caddy", []string{"int.example.com", "10.213.0.0/16"}, 90, []byte(csr)) + if err != nil { + t.Fatal(err) + } + if !c.IsCA || c.Subject.CommonName != "caddy" || len(c.PermittedDNSDomains) != 1 || c.PermittedDNSDomains[0] != "int.example.com" || + len(c.PermittedIPRanges) != 1 || c.PermittedIPRanges[0].String() != "10.213.0.0/16" || !c.PermittedDNSDomainsCritical { + t.Fatalf("constraints: %+v %v %v", c.Subject, c.PermittedDNSDomains, c.PermittedIPRanges) + } + if d := c.NotAfter.Sub(c.NotBefore); d < 89*24*time.Hour || d > 91*24*time.Hour { + t.Fatalf("validity %v", d) + } + if _, _, err := a.signIntermediate("caddy", []string{"int.example.com"}, 90, []byte("not a csr")); err == nil { + t.Fatal("signed something that is not a CSR") + } + + // due in the last third of its life, or when another root signs now + x := &Intermediate{Name: "caddy", cert: pemCert, Issuer: fpHex(ca.Cert)} + if x.due(fpHex(ca.Cert), time.Now()) { + t.Fatal("due right after signing") + } + if !x.due(fpHex(ca.Cert), time.Now().Add(61*24*time.Hour)) { + t.Fatal("not due with a third of its life left") + } + if !x.due("another-root", time.Now()) { + t.Fatal("not due after a rotation") + } +} + +func TestIntermediateProblems(t *testing.T) { + now := time.Now() + ago := func(d time.Duration) *time.Time { x := now.Add(-d); return &x } + for _, c := range []struct { + x Intermediate + want string // a fragment, "" for none + }{ + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-time.Hour)}, ""}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour), LastSeen: ago(time.Hour)}, ""}, + {Intermediate{NotAfter: now.Add(10 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expires in 10 days"}, + {Intermediate{NotAfter: now.Add(-time.Hour), Issuer: "r", LastSeen: ago(time.Hour)}, "expired"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", Created: now.Add(-100 * time.Hour)}, "never asked"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(100 * time.Hour)}, "has not asked since"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "r", LastSeen: ago(time.Hour), LastError: "CSR signature"}, "last renewal failed"}, + {Intermediate{NotAfter: now.Add(200 * 24 * time.Hour), Issuer: "old", LastSeen: ago(time.Hour)}, "no longer signs"}, + } { + got := c.x.problem("r", now) + if (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v: %q, want %q", c.x, got, c.want) + } + } + if hashToken("a") == hashToken("b") || len(hashToken("a")) != 64 { + t.Fatal("hashToken") + } + if got := splitList(" a, ,b ,"); len(got) != 2 || got[0] != "a" || got[1] != "b" { + t.Fatalf("splitList %q", got) + } +} diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 63c92f9..6623146 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -32,7 +32,8 @@ func usage() { rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) - rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA + rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA (the region renews it) + rollcall ca intermediates the intermediates the region keeps, and what needs attention rollcall ca rotate [status | prepare | switch | retire] [-force] replace the root without re-joining racks rollcall version rollcall keygen diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 11bac09..f0b743a 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -252,6 +252,10 @@ func (a *App) routes() http.Handler { mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete)) mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM)) mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation)) + mux.HandleFunc("GET /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("POST /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("DELETE /api/v1/ca/intermediates", a.adminAuth(a.handleIntermediates)) + mux.HandleFunc("POST /api/v1/ca/intermediates/{name}/renew", a.handleIntermediateRenew) // its own token (intermediates.go) mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS)) mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync)) mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts)) diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index a0b0ea3..ad2306f 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -115,6 +115,19 @@ CREATE TABLE IF NOT EXISTS switches ( source text NOT NULL DEFAULT 'local' ); ALTER TABLE switches ADD COLUMN IF NOT EXISTS role text; +CREATE TABLE IF NOT EXISTS intermediates ( + name text PRIMARY KEY, + permits text NOT NULL, + days int NOT NULL, + cert text NOT NULL, + not_after timestamptz NOT NULL, + issuer text NOT NULL, + token_hash text NOT NULL, + created timestamptz NOT NULL DEFAULT now(), + renewed timestamptz, + last_seen timestamptz, + last_error text +); CREATE TABLE IF NOT EXISTS cables ( machine_id text NOT NULL REFERENCES machines(id) ON DELETE CASCADE, port text NOT NULL, diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index be3f5b5..3f79b81 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -91,7 +91,7 @@ async function refresh(full) { $('#n-net').textContent = off ? `${off} offline` : (subnets.length || ''); const page = route().page; if (full || page === 'overview' || page === 'jobs') S.jobs = await api('GET', '/api/v1/jobs'); - if (full || page === 'overview') S.events = await api('GET', '/api/v1/events'); + if (full || page === 'overview') [S.events, S.inter] = await Promise.all([api('GET', '/api/v1/events'), api('GET', '/api/v1/ca/intermediates').catch(() => [])]); if (full || page === 'sites') [S.sites, S.rooms, S.cabling] = await Promise.all([api('GET', '/api/v1/sites'), api('GET', '/api/v1/rooms'), api('GET', '/api/v1/cabling').catch(() => null)]); if (full || page === 'overview' || page === 'network') { [S.ips, S.ipam] = await Promise.all([api('GET', '/api/v1/ips'), api('GET', '/api/v1/ipam').catch(() => null)]); @@ -181,7 +181,10 @@ function pageOverview() { ['Jobs failed (last 100)', failed.length ? `${failed.length}` : '0'], ['Oldest ready', (() => { const r = ready.slice().sort((a, b) => new Date(a.last_seen) - new Date(b.last_seen))[0]; return r ? `${esc(nameOf(r))} · ${ago(r.last_seen)}` : '—'; })()]])}`; // attention + const inter = S.inter || [], interBad = inter.filter(x => x.problem); const attention = kv([ + ...(inter.length ? [['Intermediate CAs', interBad.length ? interBad.map(x => `${esc(x.name)}: ${esc(x.problem)}`).join('
') + : `${inter.length} renewed by their hosts (next end ${esc(inter.map(x => x.not_after).sort()[0].slice(0, 10))})`]] : []), ['Rack controllers', offline.length ? `${offline.map(c => esc(c.name)).join(', ')} offline show` : `${S.controllers.length} online`], ['External IPAM', !S.ipam || !S.ipam.provider ? 'built-in only' : ipamLast && ipamLast.error ? `${esc(ipamLast.error.slice(0, 80))}` : `${esc(S.ipam.provider)} in sync${ipamLast && ipamLast.conflicts && ipamLast.conflicts.length ? ` ${ipamLast.conflicts.length} conflicts` : ''} ${ipamLast && ipamLast.at ? ago(ipamLast.at) + ' ago' : ''}`], ['BMC not configured', bmcBad ? `${bmcBad} show` : 'none'], diff --git a/deploy/rollcall-intermediate-renew.sh b/deploy/rollcall-intermediate-renew.sh new file mode 100644 index 0000000..c17a5a2 --- /dev/null +++ b/deploy/rollcall-intermediate-renew.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# rollcall-intermediate-renew [-force] NAME: keep a delegated intermediate CA +# current. Runs daily on the host that holds the intermediate's key (an ACME +# server: Caddy, step-ca), from /etc/rollcall-intermediate/NAME.env: +# URL=https://region.example the region (HTTPS) +# TOKEN_FILE=/etc/rollcall-intermediate/NAME.token from `rollcall ca intermediate` +# CERT=/etc/caddy/pki/int.pem the intermediate's certificate (replaced) +# KEY=/etc/caddy/pki/int-key.pem its key (stays; the CSR is made from it) +# ROOT=/etc/caddy/pki/root.pem optional: kept to the active root +# CA=/etc/rollcall-intermediate/NAME-bundle.pem the region's roots (seeded from ROOT, kept current) +# RELOAD='systemctl restart caddy' optional: after a new certificate +# The region decides whether a new certificate is due (the last third of its +# life, or a root rotation) and signs with the constraints it was given. +set -eu +force= +[ "${1:-}" = -force ] && { force=1; shift; } +NAME=${1:?usage: rollcall-intermediate-renew [-force] NAME} +. "/etc/rollcall-intermediate/$NAME.env" +: "${URL:?}" "${TOKEN_FILE:?}" "${CERT:?}" "${KEY:?}" +CA=${CA:-/etc/rollcall-intermediate/$NAME-bundle.pem} +if [ ! -s "$CA" ]; then + [ -n "${ROOT:-}" ] && [ -s "$ROOT" ] || { echo "rollcall: $CA missing and no ROOT to start it from" >&2; exit 1; } + cp "$ROOT" "$CA" +fi +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT + +# the region's roots, checked against the ones held: a rotation reaches here during prepare +if curl -fsS --max-time 30 --cacert "$CA" -o "$T/bundle" "$URL/ca-bundle.pem" && grep -q 'BEGIN CERTIFICATE' "$T/bundle"; then + cmp -s "$T/bundle" "$CA" || { cat "$T/bundle" > "$CA"; echo "rollcall: $NAME: the region's roots changed; $CA updated"; } +fi + +openssl req -new -key "$KEY" -subj "/CN=$NAME" -out "$T/csr" 2>/dev/null +q=; [ -n "$force" ] && q='?force=1' +code=$(curl -sS --max-time 60 --cacert "$CA" -o "$T/cert" -w '%{http_code}' -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ + -H 'Accept: application/x-pem-file' --data-binary @"$T/csr" "$URL/api/v1/ca/intermediates/$NAME/renew$q") || { echo "rollcall: $NAME: the region did not answer" >&2; exit 1; } +case $code in + 204) exit 0 ;; # nothing due + 200) ;; + *) echo "rollcall: $NAME: renewal refused (HTTP $code): $(cat "$T/cert")" >&2; exit 1 ;; +esac + +# what came back: a CA certificate for this key, from a root held here +openssl x509 -in "$T/cert" -noout -text | grep -q 'CA:TRUE' || { echo "rollcall: $NAME: the answer is not a CA certificate" >&2; exit 1; } +[ "$(openssl x509 -in "$T/cert" -noout -pubkey)" = "$(openssl pkey -in "$KEY" -pubout)" ] || { echo "rollcall: $NAME: the certificate is not for $KEY" >&2; exit 1; } +openssl verify -CAfile "$CA" "$T/cert" >/dev/null || { echo "rollcall: $NAME: the certificate does not verify against $CA" >&2; exit 1; } +put() { # $1 new content file, $2 target: same owner and mode, the old one kept as .prev + [ -f "$2" ] && cp -p "$2" "$2.prev" + if [ -f "$2" ]; then cat "$1" > "$2"; else install -m 0644 "$1" "$2"; fi +} +put "$T/cert" "$CERT" +if [ -n "${ROOT:-}" ]; then # the active root: the first certificate of the bundle + awk '/-----BEGIN CERTIFICATE-----/{n++} n==1{print} /-----END CERTIFICATE-----/ && n==1{exit}' "$CA" > "$T/root" + cmp -s "$T/root" "$ROOT" || put "$T/root" "$ROOT" +fi +echo "rollcall: $NAME renewed: $(openssl x509 -in "$CERT" -noout -enddate)" +[ -n "${RELOAD:-}" ] && sh -c "$RELOAD" +exit 0 diff --git a/deploy/rollcall-intermediate-renew@.service b/deploy/rollcall-intermediate-renew@.service new file mode 100644 index 0000000..0f946dc --- /dev/null +++ b/deploy/rollcall-intermediate-renew@.service @@ -0,0 +1,8 @@ +[Unit] +Description=rollcall: renew the delegated intermediate CA %i +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/rollcall-intermediate-renew %i diff --git a/deploy/rollcall-intermediate-renew@.timer b/deploy/rollcall-intermediate-renew@.timer new file mode 100644 index 0000000..aefd053 --- /dev/null +++ b/deploy/rollcall-intermediate-renew@.timer @@ -0,0 +1,11 @@ +[Unit] +Description=rollcall: renew the delegated intermediate CA %i daily + +[Timer] +OnBootSec=15min +OnUnitActiveSec=1d +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target