diff --git a/CHANGELOG.md b/CHANGELOG.md index d5def79..e4a8710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ publish a tag that has no section here. ## [Unreleased] +### Added +- Root rotation without re-joining (`rollcall ca rotate prepare | switch | retire`, status without + an argument). The trust bundle (all current roots, `/ca-bundle.pem`) reaches rack controllers + signed by the active root and is installed by their root unit only when that signature verifies; + racks renew client and server certificates from the new root by themselves; `switch` and + `retire` wait for every rack. Deployed machines get one file per root and a daily + `rollcall-ca-refresh` timer that fetches the bundle from their rack over HTTPS. + Upgrading: rerun `deploy/install-rack.sh` on rack hosts (the update unit installs bundles now). + +### Changed +- `rollcall ca rotate -yes` (replace the root, racks join again) is gone; the three steps replace it. + ## [0.3.0] - 2026-10-30 Third release: working with a site's existing network services (DNS provider, fixed leases on diff --git a/README.md b/README.md index 1d52de5..d6ae315 100644 --- a/README.md +++ b/README.md @@ -558,10 +558,34 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and short-lived certificates to nodes with certbot or acme.sh; everything that trusts the Rollcall CA trusts them, and nothing outside the permitted names can be issued. Roots created before this (path length 0) cannot delegate: - `rollcall ca rotate -yes` replaces the root, keeping the old one as a - backup; rack controllers then join again and clients trust the new - fingerprint. `deploy/tags/acme-cert.yaml` is a tag that fetches a node - certificate at first boot and renews it. + rotate the root (below). `deploy/tags/acme-cert.yaml` is a tag that fetches + a node certificate at first boot and renews it. +* **Root rotation without re-joining.** In three steps, each followed by a + restart of the region (`systemctl restart rollcall`): + + ``` + rollcall ca rotate prepare # a new root; the old one still signs + rollcall ca rotate # status: which rack has what + rollcall ca rotate switch # the new root signs; the old one stays trusted + rollcall ca rotate retire # the old root is no longer trusted + ``` + + Everything that trusts the region gets the *trust bundle* (all current + roots, `/ca-bundle.pem`). Rack controllers receive it signed by the active + root and their root unit installs it only if that signature verifies + against the roots they already trust; after `switch` they renew their + client and server certificates from the new root by themselves. `switch` + waits until every rack has the bundle, `retire` until every rack runs on + certificates from the new root (`-force` goes on; racks that missed it + join again). Machines deployed by this version get the roots one per file + and a daily `rollcall-ca-refresh` timer that fetches the bundle over HTTPS + from their rack, checked against the roots they trust, so they pick up a + new root during `prepare` and drop the old one after `retire`. Machines + deployed earlier need the bundle installed once by hand + (`curl --cacert https:///ca-bundle.pem`). Admin + certificates and intermediates signed by the old root work until `retire`; + issue them again after `switch`. `/ca.pem` stays a single certificate (the + root of that server's own certificate), which agents check by fingerprint. ## Power drivers diff --git a/cmd/rollcall/admincert.go b/cmd/rollcall/admincert.go index cc2aa3e..2ffa781 100644 --- a/cmd/rollcall/admincert.go +++ b/cmd/rollcall/admincert.go @@ -214,7 +214,7 @@ func (a *App) handleAdminCerts(w http.ResponseWriter, r *http.Request) { } a.dropAdminCertCache() log.Printf("admin certificate %q (serial %s, until %s) issued by %s", req.Name, cert.SerialNumber.Text(16), cert.NotAfter.Format("2006-01-02"), by) - writeJSON(w, 200, obj{"name": req.Name, "serial": cert.SerialNumber.Text(16), "expires": cert.NotAfter, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": req.Name, "serial": cert.SerialNumber.Text(16), "expires": cert.NotAfter, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } } diff --git a/cmd/rollcall/agenttls.go b/cmd/rollcall/agenttls.go index 0d9ab57..0859701 100644 --- a/cmd/rollcall/agenttls.go +++ b/cmd/rollcall/agenttls.go @@ -68,7 +68,7 @@ func (a *App) handleRackServerCert(w http.ResponseWriter, r *http.Request, rack return } log.Printf("rack controller %s: server certificate for %s until %s", rack, host, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // rackHost is the host part of a rack URL (an address or a name). @@ -157,7 +157,7 @@ func (a *App) agentTLSArgs(base string) []string { if t == "" { return nil } - fp := certFingerprint(a.ca.Cert) + fp := certFingerprint(a.serverCAFor(base)) // the root the rack serves as /ca.pem return []string{"rollcall.tls=" + t, "rollcall.ca=" + hex.EncodeToString(fp[:])} } diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index 7465cc0..78751cc 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -55,7 +55,11 @@ const ( type CA struct { Cert *x509.Certificate Key *ecdsa.PrivateKey - PEM []byte + PEM []byte // the active root + // all trusted roots, active first: during a rotation also the next or + // previous one (carot.go); what clients and machines are given to trust + Trust []*x509.Certificate + TrustPEM []byte } func randomSerial() (*big.Int, error) { @@ -121,7 +125,11 @@ func loadOrCreateCA(dir, cn string) (*CA, error) { if err != nil { return nil, fmt.Errorf("%s: %w", kp, err) } - return &CA{Cert: cert, Key: key, PEM: cb}, nil + ca := &CA{Cert: cert, Key: key, PEM: cb} + if err := ca.loadTrust(dir); err != nil { + return nil, err + } + return ca, nil } ca, err := newCA(cn) if err != nil { @@ -138,6 +146,7 @@ func loadOrCreateCA(dir, cn string) (*CA, error) { return nil, err } log.Printf("created certificate authority %q in %s (fingerprint %s)", cn, dir, ca.Fingerprint()) + ca.Trust, ca.TrustPEM = []*x509.Certificate{ca.Cert}, ca.PEM return ca, nil } @@ -609,7 +618,7 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) { a.reloadRacks(r.Context()) a.hub.bump() log.Printf("rack controller %s enrolled from %s, certificate until %s", name, r.RemoteAddr, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // handleRackRenew issues a new certificate to a rack controller that still has a valid one. @@ -633,7 +642,7 @@ func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack strin return } log.Printf("rack controller %s renewed its certificate, until %s", rack, cert.NotAfter.Format("2006-01-02")) - writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.PEM)}) + writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.TrustPEM)}) } // ---------------------------------------------------------------- CLI: rollcall ca (on the region host) @@ -654,6 +663,13 @@ func caCmd(args []string) error { } fmt.Printf("%s\n subject %s\n valid until %s\n fingerprint %s\n", filepath.Join(cfg.CADir, "ca.pem"), c.Subject.CommonName, c.NotAfter.Format("2006-01-02"), fpString(certFingerprint(c))) + for _, f := range []string{"ca-next.pem", "ca-prev.pem"} { + if b, err := os.ReadFile(filepath.Join(cfg.CADir, f)); err == nil { + if o, err := parseCertPEM(b); err == nil { + fmt.Printf("%s (also trusted, rotation)\n fingerprint %s\n", f, fpString(certFingerprint(o))) + } + } + } if rb, err := os.ReadFile(filepath.Join(cfg.CADir, "region.pem")); err == nil { if rc, err := parseCertPEM(rb); err == nil { names := append([]string{}, rc.DNSNames...) @@ -777,38 +793,6 @@ func caIntermediateCmd(cfg *Config, name string, args []string) error { return nil } -// rollcall ca rotate -yes: a new root (the old one is kept as a backup). Racks -// must join again and clients trust the new fingerprint. -func caRotateCmd(cfg *Config, args []string) error { - fs := newFlags("ca rotate") - yes := fs.Bool("yes", false, "really replace the root") - _ = fs.Parse(args) - old, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - return err - } - if !*yes { - return fmt.Errorf("this replaces the root %s (delegates: %v); rack controllers then join again and clients trust the new one. Run with -yes", old.Fingerprint(), old.canDelegate()) - } - bak := cfg.CADir + ".bak-" + time.Now().Format("20060102-150405") - if err := os.Rename(cfg.CADir, bak); err != nil { - return err - } - ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA") - if err != nil { - _ = os.Rename(bak, cfg.CADir) - return err - } - if st, err := os.Stat(bak); err == nil { - _ = os.Chmod(cfg.CADir, st.Mode().Perm()) - if own, ok := fileOwner(bak); ok { - _ = chownTree(cfg.CADir, own[0], own[1]) - } - } - fmt.Printf("new root %s\nold root kept in %s\nnext: restart rollcall; on each rack controller `rollcall rack join` with a new join code; clients trust the new fingerprint\n", ca.Fingerprint(), bak) - return nil -} - // fileOwner returns uid, gid of path (unix). func fileOwner(path string) ([2]int, bool) { st, err := os.Stat(path) diff --git a/cmd/rollcall/carot.go b/cmd/rollcall/carot.go new file mode 100644 index 0000000..9ffe1a6 --- /dev/null +++ b/cmd/rollcall/carot.go @@ -0,0 +1,673 @@ +package main + +// Root rotation without re-joining. The CA directory holds the active root +// (ca.pem, ca-key.pem) and, during a rotation, the next one (ca-next.pem, +// ca-next-key.pem) or the previous one (ca-prev.pem). Everything that trusts +// the region gets the trust bundle (all of them, active first): +// +// prepare a new root is made; the old one still signs. Rack controllers +// install the bundle (signed by the active root, installed by their +// root unit), machines deployed from now on trust both. +// switch the new root signs: the region's certificate, rack and admin +// certificates from now on. Racks renew their client and server +// certificates by themselves; the old root stays trusted. +// retire the old root leaves the bundle once every rack runs on the new one. +// +// switch and retire refuse while a rack is not ready (-force overrides). +// /ca.pem stays one certificate (the root of the server's own certificate: +// agents in older images read one); /ca-bundle.pem is the bundle. + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// loadTrust adds the next and previous roots of a rotation to ca.Trust. +func (ca *CA) loadTrust(dir string) error { + ca.Trust = []*x509.Certificate{ca.Cert} + ca.TrustPEM = append([]byte{}, ca.PEM...) + for _, f := range []string{"ca-next.pem", "ca-prev.pem"} { + b, err := os.ReadFile(filepath.Join(dir, f)) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + c, err := parseCertPEM(b) + if err != nil || !c.IsCA { + return fmt.Errorf("%s: not a CA certificate", f) + } + ca.Trust = append(ca.Trust, c) + ca.TrustPEM = append(ca.TrustPEM, pemEncode("CERTIFICATE", c.Raw)...) + } + return nil +} + +func fpHex(c *x509.Certificate) string { + sum := certFingerprint(c) + return hex.EncodeToString(sum[:]) +} + +func (ca *CA) pool() *x509.CertPool { + p := x509.NewCertPool() + for _, c := range ca.Trust { + p.AddCert(c) + } + return p +} + +// signBytes is the active root's signature over data's SHA-256 (base64 DER), +// as for rack binaries: openssl dgst -sha256 -verify checks it. +func (ca *CA) signBytes(data []byte) string { + sum := sha256.Sum256(data) + der, err := ecdsa.SignASN1(rand.Reader, ca.Key, sum[:]) + if err != nil { + return "" + } + return base64.StdEncoding.EncodeToString(der) +} + +// rootOf is the root among trust that signed c, nil when none did. +func rootOf(c *x509.Certificate, trust []*x509.Certificate) *x509.Certificate { + if c == nil { + return nil + } + for _, r := range trust { + if bytes.Equal(c.RawIssuer, r.RawSubject) && c.CheckSignatureFrom(r) == nil { + return r + } + } + return nil +} + +// verifyBundle: a bundle the region sent, signed by one of the roots trusted +// now, holding CA certificates only. +func verifyBundle(bundle []byte, sig string, trust []*x509.Certificate) ([]*x509.Certificate, error) { + der, err := base64.StdEncoding.DecodeString(sig) + if err != nil { + return nil, fmt.Errorf("bundle signature: %w", err) + } + sum := sha256.Sum256(bundle) + ok := false + for _, r := range trust { + if pub, isEC := r.PublicKey.(*ecdsa.PublicKey); isEC && ecdsa.VerifyASN1(pub, sum[:], der) { + ok = true + } + } + if !ok { + return nil, errors.New("the bundle is not signed by a root trusted here") + } + certs, err := parsePEMCerts(bundle) + if err != nil { + return nil, err + } + for _, c := range certs { + if !c.IsCA { + return nil, fmt.Errorf("bundle: %q is not a CA certificate", c.Subject.CommonName) + } + } + return certs, nil +} + +func parsePEMCerts(b []byte) ([]*x509.Certificate, error) { + var out []*x509.Certificate + for { + var blk *pem.Block + blk, b = pem.Decode(b) + if blk == nil { + break + } + if blk.Type != "CERTIFICATE" { + return nil, fmt.Errorf("unexpected PEM block %s", blk.Type) + } + c, err := x509.ParseCertificate(blk.Bytes) + if err != nil { + return nil, err + } + out = append(out, c) + } + if len(out) == 0 { + return nil, errors.New("no certificate") + } + return out, nil +} + +// ---------------------------------------------------------------- store + +// SetRackTrust records what a rack runs on: the hash of its trust bundle and +// the roots of its client and server certificates. +func (s *Store) SetRackTrust(ctx context.Context, name, bundleSHA, clientCA, serverCA string) error { + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET ca_bundle_sha256=nullif($2,''), client_ca=nullif($3,''), + server_ca=coalesce(nullif($4,''), server_ca) WHERE name=$1`, name, bundleSHA, clientCA, serverCA) + return err +} + +type rackTrust struct { + Name string `json:"name"` + Bundle string `json:"ca_bundle_sha256"` + ClientCA string `json:"client_ca"` + ServerCA string `json:"server_ca"` + LastSeen *time.Time `json:"last_seen"` +} + +func (s *Store) RackTrust(ctx context.Context) ([]rackTrust, error) { + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(ca_bundle_sha256,''), coalesce(client_ca,''), coalesce(server_ca,''), last_seen + FROM rack_controllers WHERE kind='remote' ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []rackTrust{} + for rows.Next() { + var t rackTrust + if err := rows.Scan(&t.Name, &t.Bundle, &t.ClientCA, &t.ServerCA, &t.LastSeen); err != nil { + return nil, err + } + out = append(out, t) + } + return out, rows.Err() +} + +// ---------------------------------------------------------------- region HTTP + +func (a *App) handleCABundle(w http.ResponseWriter, r *http.Request) { + if a.ca == nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-pem-file") + _, _ = w.Write(a.ca.TrustPEM) +} + +type rotationStatus struct { + Phase string `json:"phase"` // "" | prepared | switched + Active string `json:"active"` + Next string `json:"next,omitempty"` + Prev string `json:"prev,omitempty"` + Bundle string `json:"bundle_sha256"` + Racks []rackTrust `json:"racks"` + // what blocks the next step + NotReady []string `json:"not_ready"` +} + +func (a *App) rotation(ctx context.Context) (rotationStatus, error) { + st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}} + sum := sha256.Sum256(a.ca.TrustPEM) + st.Bundle = hex.EncodeToString(sum[:]) + if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-next.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + st.Phase, st.Next = "prepared", fpHex(c) + } + } + if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil { + if c, err := parseCertPEM(b); err == nil { + st.Phase, st.Prev = "switched", fpHex(c) + } + } + racks, err := a.store.RackTrust(ctx) + if err != nil { + return st, err + } + st.Racks = racks + for _, r := range racks { + switch st.Phase { + case "prepared": // switch needs the new root at every rack + if r.Bundle != st.Bundle { + st.NotReady = append(st.NotReady, r.Name+": has not installed the bundle with the new root") + } + case "switched": // retire needs every rack on certificates from the new root + if r.ClientCA != st.Active || r.ServerCA != "" && r.ServerCA != st.Active { + st.NotReady = append(st.NotReady, r.Name+": still has a certificate from the old root") + } + } + } + return st, nil +} + +func (a *App) handleCARotation(w http.ResponseWriter, r *http.Request) { + if a.ca == nil { + fail(w, 409, errors.New("no CA")) + return + } + st, err := a.rotation(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, st) +} + +// noteRackTrust records, at registration, the root of the client certificate +// the rack connected with and what it says about its bundle and server certificate. +func (a *App) noteRackTrust(r *http.Request, rack string, rc RackController) { + if a.ca == nil || r.TLS == nil { + return + } + var client string + for _, ch := range r.TLS.VerifiedChains { + if len(ch) == 2 { + client = fpHex(ch[1]) + } + } + _ = a.store.SetRackTrust(r.Context(), rack, rc.CABundleSHA256, client, rc.ServerCA) +} + +// serverCAFor is the root an agent behind this rack must accept: that of the +// rack's server certificate (it serves that one as /ca.pem). +func (a *App) serverCAFor(base string) *x509.Certificate { + if base != a.cfg.ServerURL { + for _, rc := range a.racksSnapshot() { + if strings.TrimRight(rc.URL, "/") == base && rc.ServerCA != "" { + for _, c := range a.ca.Trust { + if fpHex(c) == rc.ServerCA { + return c + } + } + } + } + } + return a.ca.Cert +} + +var trustSig struct { + sync.Mutex + bundle, sig string +} + +// trustState is signed once per bundle: the rack state stays the same between polls. +func (a *App) trustState() *rackTrustState { + trustSig.Lock() + defer trustSig.Unlock() + if trustSig.bundle != string(a.ca.TrustPEM) { + trustSig.bundle, trustSig.sig = string(a.ca.TrustPEM), a.ca.signBytes(a.ca.TrustPEM) + } + return &rackTrustState{Bundle: trustSig.bundle, Sig: trustSig.sig, Active: fpHex(a.ca.Cert)} +} + +// caRefreshFiles keep a deployed machine's roots current: the bundle comes +// over HTTPS from the rack, checked against the roots the machine trusts now, +// so a rotation reaches it during prepare (before the new root signs) and the +// old root leaves it after retire. +func caRefreshFiles(tlsBase string) []seedFile { + script := `#!/bin/sh +# rollcall: keep the installation's roots current (written at deploy time) +set -eu +URL=` + shQuote(tlsBase+"/ca-bundle.pem") + ` +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +curl -fsS --max-time 30 -o "$T/bundle" "$URL" || exit 0 # unreachable: the timer tries again +awk -v d="$T" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n)} /-----END CERTIFICATE-----/{close(d "/c" n)}' "$T/bundle" +[ -f "$T/c1" ] || exit 0 +changed= +for dir in /usr/local/share/ca-certificates /etc/pki/ca-trust/source/anchors; do + [ -d "$dir" ] || continue + i=0; mkdir -p "$T/want"; rm -f "$T"/want/* + for c in "$T"/c[0-9]*; do + i=$((i+1)); n=rollcall-ca.crt; [ $i -gt 1 ] && n=rollcall-ca-$i.crt + cp "$c" "$T/want/$n" + done + if [ "$(cat "$dir"/rollcall-ca*.crt 2>/dev/null)" != "$(cat "$T"/want/*)" ]; then + rm -f "$dir"/rollcall-ca*.crt + for f in "$T"/want/*; do install -m 0644 "$f" "$dir/$(basename "$f")"; done + changed=1 + fi +done +[ -n "$changed" ] || exit 0 +if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi +echo "rollcall: trusted roots updated from $URL" +` + return []seedFile{ + {Path: "usr/local/sbin/rollcall-ca-refresh", Mode: "0755", Content: script}, + {Path: "etc/systemd/system/rollcall-ca-refresh.service", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/rollcall-ca-refresh\n"}, + {Path: "etc/systemd/system/rollcall-ca-refresh.timer", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots daily\n\n[Timer]\nOnBootSec=10min\nOnUnitActiveSec=1d\nRandomizedDelaySec=1h\n\n[Install]\nWantedBy=timers.target\n"}, + } +} + +// ---------------------------------------------------------------- CLI + +// rollcall ca rotate [status | prepare | switch | retire] [-force] +func caRotateCmd(cfg *Config, args []string) error { + sub := "status" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + sub, args = args[0], args[1:] + } + fs := newFlags("ca rotate " + sub) + force := fs.Bool("force", false, "go on although rack controllers are not ready (they then join again)") + _ = fs.Parse(args) + dir := cfg.CADir + have := func(f string) bool { _, err := os.Stat(filepath.Join(dir, f)); return err == nil } + status := func() (*rotationStatus, error) { + c, err := newClient() + if err != nil { + return nil, err + } + var st rotationStatus + if err := c.do("GET", "/api/v1/ca/rotation", nil, &st); err != nil { + return nil, err + } + return &st, nil + } + ready := func() error { + st, err := status() + if err != nil { + if *force { + fmt.Fprintf(os.Stderr, "cannot ask the region (%v); -force: going on\n", err) + return nil + } + return fmt.Errorf("cannot ask the region whether rack controllers are ready: %w (-force to go on)", err) + } + if len(st.NotReady) > 0 && !*force { + return fmt.Errorf("not ready:\n %s\nwait for them (they pick it up within minutes when online) or -force (they then join again)", strings.Join(st.NotReady, "\n ")) + } + return nil + } + restart := "next: restart rollcall (systemctl restart rollcall); `rollcall ca rotate` shows the progress" + switch sub { + case "status": + st, err := status() + if err != nil { + return err + } + phase := map[string]string{"": "no rotation", "prepared": "prepared (next: switch)", "switched": "switched (next: retire)"}[st.Phase] + fmt.Printf("phase %s\nactive %s\n", phase, fpString(mustFP(st.Active))) + if st.Next != "" { + fmt.Printf("next %s\n", fpString(mustFP(st.Next))) + } + if st.Prev != "" { + fmt.Printf("prev %s (still trusted)\n", fpString(mustFP(st.Prev))) + } + short := func(fp string) string { + switch { + case fp == "": + return "-" + case fp == st.Active: + return "active" + case fp == st.Next: + return "next" + case fp == st.Prev: + return "prev" + } + return fp[:12] + } + fmt.Printf("\n%-16s %-8s %-8s %-8s %s\n", "RACK", "BUNDLE", "CLIENT", "SERVER", "LAST SEEN") + for _, r := range st.Racks { + b := "old" + if r.Bundle == st.Bundle { + b = "current" + } + seen := "-" + if r.LastSeen != nil { + seen = r.LastSeen.Local().Format("2006-01-02 15:04") + } + fmt.Printf("%-16s %-8s %-8s %-8s %s\n", r.Name, b, short(r.ClientCA), short(r.ServerCA), seen) + } + for _, n := range st.NotReady { + fmt.Println("waiting:", n) + } + return nil + case "prepare": + if have("ca-next.pem") || have("ca-prev.pem") { + return errors.New("a rotation is under way (rollcall ca rotate status)") + } + next, err := newCA("rollcall CA " + time.Now().Format("2006-01-02")) + if err != nil { + return err + } + if err := writeCA(dir, "ca-next", next); err != nil { + return err + } + fmt.Printf("next root %s\nthe active root still signs; rack controllers and newly deployed machines get both\n%s; then `rollcall ca rotate switch`\n", next.Fingerprint(), restart) + return nil + case "switch": + if !have("ca-next.pem") { + return errors.New("no prepared root (rollcall ca rotate prepare)") + } + if err := ready(); err != nil { + return err + } + bak := filepath.Join(dir, "rotated-"+time.Now().Format("20060102-150405")) + if err := os.Mkdir(bak, 0o700); err != nil { + return err + } + steps := [][2]string{{"ca.pem", "ca-prev.pem"}, {"ca-key.pem", filepath.Join(filepath.Base(bak), "ca-key.pem")}, + {"ca-next-key.pem", "ca-key.pem"}, {"ca-next.pem", "ca.pem"}} + for _, s := range steps { + if err := copyKeepOwner(filepath.Join(dir, s[0]), filepath.Join(dir, s[1])); err != nil { + return fmt.Errorf("%s -> %s: %w (the CA directory may be half switched; the old files are in %s)", s[0], s[1], err, bak) + } + } + for _, f := range []string{"ca-next.pem", "ca-next-key.pem"} { + _ = os.Remove(filepath.Join(dir, f)) + } + fmt.Printf("the new root signs now; the old one stays trusted (its key: %s)\nrack controllers renew their certificates by themselves; admin certificates and intermediates from the old root work until retire: issue them again\n%s; then `rollcall ca rotate retire`\n", bak, restart) + return nil + case "retire": + if !have("ca-prev.pem") { + return errors.New("no previous root to retire") + } + if err := ready(); err != nil { + return err + } + bak := filepath.Join(dir, "retired-"+time.Now().Format("20060102-150405")+".pem") + if err := os.Rename(filepath.Join(dir, "ca-prev.pem"), bak); err != nil { + return err + } + fmt.Printf("the old root is no longer trusted (kept in %s)\n%s\n", bak, restart) + return nil + } + return fmt.Errorf("ca rotate %s: unknown (status, prepare, switch, retire)", sub) +} + +func mustFP(h string) [32]byte { + var out [32]byte + b, _ := hex.DecodeString(h) + copy(out[:], b) + return out +} + +// writeCA writes a root as .pem / -key.pem, owned like ca.pem. +func writeCA(dir, name string, ca *CA) error { + kder, err := x509.MarshalECPrivateKey(ca.Key) + if err != nil { + return err + } + kp, cp := filepath.Join(dir, name+"-key.pem"), filepath.Join(dir, name+".pem") + if err := os.WriteFile(kp, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil { + return err + } + if err := os.WriteFile(cp, ca.PEM, 0o644); err != nil { + return err + } + if own, ok := fileOwner(filepath.Join(dir, "ca.pem")); ok { + _ = os.Chown(kp, own[0], own[1]) + _ = os.Chown(cp, own[0], own[1]) + } + return nil +} + +// copyKeepOwner copies src over dst (atomically) with src's mode and owner. +func copyKeepOwner(src, dst string) error { + b, err := os.ReadFile(src) + if err != nil { + return err + } + st, err := os.Stat(src) + if err != nil { + return err + } + if err := writeAtomic(dst, b, st.Mode().Perm()); err != nil { + return err + } + if own, ok := fileOwner(src); ok { + _ = os.Chown(dst, own[0], own[1]) + } + return nil +} + +// shQuote is s as one single-quoted shell word. +func shQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } + +// ---------------------------------------------------------------- rack controller + +// rackTrust is what the rack trusts now (its ca_file, written at join and by +// the root update unit). +func (r *rackd) rackTrust() ([]byte, []*x509.Certificate) { + b, err := os.ReadFile(r.cfg.CAFile) + if err != nil { + return nil, nil + } + certs, _ := parsePEMCerts(b) + return b, certs +} + +// trustReport: the hash of the rack's roots and the root of its server certificate. +func (r *rackd) trustReport() (bundleSHA, serverCA string) { + b, trust := r.rackTrust() + if b == nil { + return "", "" + } + sum := sha256.Sum256(b) + r.srvMu.Lock() + c := r.srvCert + r.srvMu.Unlock() + if c != nil { + if root := rootOf(c.Leaf, trust); root != nil { + serverCA = fpHex(root) + } + } + return hex.EncodeToString(sum[:]), serverCA +} + +// serveCA answers /ca.pem with the root of the server certificate (agents +// pin it by fingerprint and read one certificate) and /ca-bundle.pem with all. +func (r *rackd) serveCA(bundle bool) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + b, trust := r.rackTrust() + if b == nil { + http.Error(w, "no CA", 404) + return + } + w.Header().Set("Content-Type", "application/x-pem-file") + if bundle || len(trust) < 2 { + _, _ = w.Write(b) + return + } + r.srvMu.Lock() + c := r.srvCert + r.srvMu.Unlock() + root := trust[0] + if c != nil { + if o := rootOf(c.Leaf, trust); o != nil { + root = o + } + } + _, _ = w.Write(pemEncode("CERTIFICATE", root.Raw)) + } +} + +var rackTrustMu sync.Mutex +var rackTrustTried = map[string]time.Time{} + +// maybeTrust follows the region's roots: a new bundle is staged for the root +// unit, and certificates from a root that no longer signs are renewed. +func (r *rackd) maybeTrust(t *rackTrustState) { + if t == nil || r.cert == nil { + return + } + rackTrustMu.Lock() + defer rackTrustMu.Unlock() + cur, trust := r.rackTrust() + if t.Bundle != "" && t.Bundle != string(cur) { + sum := sha256.Sum256([]byte(t.Bundle)) + key := "bundle " + hex.EncodeToString(sum[:]) + if time.Since(rackTrustTried[key]) > time.Hour { + rackTrustTried[key] = time.Now() + if err := r.stageBundle(t, trust); err != nil { + log.Printf("trust bundle: %v", err) + } else { + log.Printf("new trust bundle staged in %s; rollcall-rack-update installs it", r.updateDir()) + } + } + } + var active *x509.Certificate + for _, c := range trust { + if fpHex(c) == t.Active { + active = c + } + } + if active == nil { + return // the signing root is not trusted here yet: the bundle comes first + } + renewed := false + r.certMu.Lock() + leaf := r.cert.Leaf + r.certMu.Unlock() + if rootOf(leaf, trust) != active && time.Since(rackTrustTried["client"]) > 10*time.Minute { + rackTrustTried["client"] = time.Now() + if err := r.renew(); err != nil { + log.Printf("certificate from the new root: %v", err) + } else { + renewed = true + } + } + r.srvMu.Lock() + srv := r.srvCert + r.srvMu.Unlock() + if srv != nil && rootOf(srv.Leaf, trust) != active && time.Since(rackTrustTried["server"]) > 10*time.Minute { + rackTrustTried["server"] = time.Now() + if c, err := r.newServerCert(); err != nil { + log.Printf("server certificate from the new root: %v", err) + } else { + r.srvMu.Lock() + r.srvCert = c + r.srvMu.Unlock() + renewed = true + } + } + if renewed { + for _, hc := range []*http.Client{r.hc, r.fast, r.big} { // new connections present the new certificate + if tr, ok := hc.Transport.(*http.Transport); ok { + tr.CloseIdleConnections() + } + } + if err := r.register(); err != nil { + log.Printf("register after renewal: %v", err) + } + } +} + +// stageBundle leaves a verified bundle and its signature for the root unit, +// which checks the signature again against its own copy of the roots. +func (r *rackd) stageBundle(t *rackTrustState, trust []*x509.Certificate) error { + if _, err := verifyBundle([]byte(t.Bundle), t.Sig, trust); err != nil { + return err + } + sig, _ := base64.StdEncoding.DecodeString(t.Sig) + dir := r.updateDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := writeAtomic(filepath.Join(dir, "ca-bundle.pem"), []byte(t.Bundle), 0o644); err != nil { + return err + } + if err := writeAtomic(filepath.Join(dir, "ca-bundle.sig"), sig, 0o644); err != nil { + return err + } + return writeAtomic(filepath.Join(dir, "ca-ready"), []byte("1\n"), 0o644) +} diff --git a/cmd/rollcall/carot_test.go b/cmd/rollcall/carot_test.go new file mode 100644 index 0000000..b54fe9b --- /dev/null +++ b/cmd/rollcall/carot_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func rotTestCA(t *testing.T) (string, *Config) { + t.Helper() + dir := filepath.Join(t.TempDir(), "ca") + if _, err := loadOrCreateCA(dir, "rollcall CA"); err != nil { + t.Fatal(err) + } + return dir, &Config{CADir: dir} +} + +func TestCARotationPhases(t *testing.T) { + dir, cfg := rotTestCA(t) + old, _ := loadOrCreateCA(dir, "rollcall CA") + if len(old.Trust) != 1 || !bytes.Equal(old.TrustPEM, old.PEM) { + t.Fatalf("no rotation: trust %d", len(old.Trust)) + } + leafOld, _, err := old.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + if err != nil { + t.Fatal(err) + } + // prepare: the old root still signs, both are trusted + if err := caRotateCmd(cfg, []string{"prepare"}); err != nil { + t.Fatal(err) + } + if err := caRotateCmd(cfg, []string{"prepare"}); err == nil { + t.Fatal("a second prepare during a rotation") + } + p, _ := loadOrCreateCA(dir, "rollcall CA") + if fpHex(p.Cert) != fpHex(old.Cert) || len(p.Trust) != 2 { + t.Fatalf("prepared: active %s trust %d", fpHex(p.Cert)[:8], len(p.Trust)) + } + next := p.Trust[1] + // switch (no region to ask: -force) + if err := caRotateCmd(cfg, []string{"switch", "-force"}); err != nil { + t.Fatal(err) + } + s, err := loadOrCreateCA(dir, "rollcall CA") + if err != nil { + t.Fatal(err) + } + if fpHex(s.Cert) != fpHex(next) || len(s.Trust) != 2 || fpHex(s.Trust[1]) != fpHex(old.Cert) { + t.Fatalf("switched: active should be the prepared root, the old one trusted") + } + if !s.Key.PublicKey.Equal(s.Cert.PublicKey) { + t.Fatal("the active key is not the active root's") + } + // a certificate from the old root still verifies, a new one comes from the new root + lc, _ := parseCertPEM(leafOld) + if _, err := lc.Verify(x509.VerifyOptions{Roots: s.pool(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { + t.Fatalf("old certificate after switch: %v", err) + } + nb, _, _ := s.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + nc, _ := parseCertPEM(nb) + if rootOf(nc, s.Trust) != s.Cert || rootOf(lc, s.Trust) == s.Cert { + t.Fatal("rootOf") + } + // the region's certificate is issued again by the new root + if _, err := regionCert(old, dir, []string{"region.example"}); err != nil { + t.Fatal(err) + } + pair, err := regionCert(s, dir, []string{"region.example"}) + if err != nil { + t.Fatal(err) + } + rc, _ := x509.ParseCertificate(pair.Certificate[0]) + if rc.CheckSignatureFrom(s.Cert) != nil { + t.Fatal("region certificate not from the new root after switch") + } + if err := caRotateCmd(cfg, []string{"switch", "-force"}); err == nil { + t.Fatal("switch without a prepared root") + } + // retire: only the new root + if err := caRotateCmd(cfg, []string{"retire", "-force"}); err != nil { + t.Fatal(err) + } + r, _ := loadOrCreateCA(dir, "rollcall CA") + if len(r.Trust) != 1 || fpHex(r.Cert) != fpHex(next) { + t.Fatal("retired: trust should be the new root only") + } + if _, err := lc.Verify(x509.VerifyOptions{Roots: r.pool(), KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err == nil { + t.Fatal("a certificate from the retired root still verifies") + } +} + +func TestTrustBundleSignature(t *testing.T) { + dir, cfg := rotTestCA(t) + if err := caRotateCmd(cfg, []string{"prepare"}); err != nil { + t.Fatal(err) + } + ca, _ := loadOrCreateCA(dir, "rollcall CA") + sig := ca.signBytes(ca.TrustPEM) + got, err := verifyBundle(ca.TrustPEM, sig, []*x509.Certificate{ca.Cert}) + if err != nil || len(got) != 2 { + t.Fatalf("valid bundle: %v (%d)", err, len(got)) + } + stranger, _ := newCA("someone") + if _, err := verifyBundle(ca.TrustPEM, sig, []*x509.Certificate{stranger.Cert}); err == nil { + t.Fatal("accepted a bundle signed by a root not trusted here") + } + if _, err := verifyBundle(append(append([]byte{}, ca.TrustPEM...), stranger.PEM...), sig, ca.Trust); err == nil { + t.Fatal("accepted a bundle with a root added after signing") + } + leaf, _, _ := ca.issue("x", []string{"x.example"}, x509.ExtKeyUsageServerAuth) + withLeaf := append(append([]byte{}, ca.TrustPEM...), leaf...) + if _, err := verifyBundle(withLeaf, ca.signBytes(withLeaf), ca.Trust); err == nil || !strings.Contains(err.Error(), "not a CA") { + t.Fatalf("a leaf in the bundle: %v", err) + } +} + +func TestRackStagesBundle(t *testing.T) { + dir, cfg := rotTestCA(t) + ca, _ := loadOrCreateCA(dir, "rollcall CA") + rd := t.TempDir() + caFile := filepath.Join(rd, "ca.pem") + _ = os.WriteFile(caFile, ca.PEM, 0o644) + certPEM, keyPEM, _ := ca.issue("rack:r1", nil, x509.ExtKeyUsageClientAuth) + pair, _ := tls.X509KeyPair(certPEM, keyPEM) + pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) + r := &rackd{cfg: RackConfig{Name: "r1", CAFile: caFile, UpdateDir: filepath.Join(rd, "update")}, cert: &pair} + _ = caRotateCmd(cfg, []string{"prepare"}) + ca, _ = loadOrCreateCA(dir, "rollcall CA") + app := &App{ca: ca} + r.maybeTrust(app.trustState()) + b, err := os.ReadFile(filepath.Join(rd, "update", "ca-bundle.pem")) + if err != nil || !bytes.Equal(b, ca.TrustPEM) { + t.Fatalf("bundle not staged: %v", err) + } + if _, err := os.Stat(filepath.Join(rd, "update", "ca-ready")); err != nil { + t.Fatal("no ca-ready marker") + } + // the signature file is DER, for openssl dgst -verify + sig, _ := os.ReadFile(filepath.Join(rd, "update", "ca-bundle.sig")) + if len(sig) == 0 || sig[0] != 0x30 { + t.Fatal("signature not DER") + } + // a forged bundle is not staged + _ = os.RemoveAll(filepath.Join(rd, "update")) + stranger, _ := newCA("someone") + forged := append(append([]byte{}, ca.TrustPEM...), stranger.PEM...) + rackTrustTried = map[string]time.Time{} + r.maybeTrust(&rackTrustState{Bundle: string(forged), Sig: stranger.signBytes(forged), Active: fpHex(ca.Cert)}) + if _, err := os.Stat(filepath.Join(rd, "update", "ca-ready")); err == nil { + t.Fatal("staged a bundle signed by a stranger") + } + // the rack serves one certificate as /ca.pem: agents in older images read one + _ = os.WriteFile(caFile, ca.TrustPEM, 0o644) + if _, trust := r.rackTrust(); len(trust) != 2 { + t.Fatal("rack trust") + } +} + +func TestCARefreshScript(t *testing.T) { + fs := caRefreshFiles("https://10.213.78.1") + if len(fs) != 3 || !strings.Contains(fs[0].Content, "URL='https://10.213.78.1/ca-bundle.pem'") { + t.Fatalf("%+v", fs[0]) + } + f := filepath.Join(t.TempDir(), "s") + _ = os.WriteFile(f, []byte(fs[0].Content), 0o755) + if out, err := exec.Command("sh", "-n", f).CombinedOutput(); err != nil { + t.Fatalf("sh -n: %v %s", err, out) + } + if shQuote("a'b") != `'a'\''b'` { + t.Fatal("shQuote") + } +} diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index 3f39654..28fc7b2 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -311,7 +311,8 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { // cloud-init's ca_certs runs update-ca-certificates even on RHEL-family systems // and fails there, so it is not used. frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: "#cloud-config\n# rollcall: trust the installation's certificate authority\nbootcmd:\n" + - " - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n"}) + " - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n" + + " - [sh, -c, 'if [ -f /etc/systemd/system/rollcall-ca-refresh.timer ]; then systemctl enable --now rollcall-ca-refresh.timer || true; fi']\n"}) } if merr == nil { if tf, err := a.tagFragments(context.Background(), m); err == nil { @@ -327,9 +328,19 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { {Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"}, } if a.ca != nil { // both layouts; the bootcmd above picks the one the distribution uses - files = append(files, - seedFile{Path: "usr/local/share/ca-certificates/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)}, - seedFile{Path: "etc/pki/ca-trust/source/anchors/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)}) + for i, c := range a.ca.Trust { // one root per file (update-ca-certificates reads one) + name := "rollcall-ca.crt" + if i > 0 { + name = fmt.Sprintf("rollcall-ca-%d.crt", i+1) + } + pemC := string(pemEncode("CERTIFICATE", c.Raw)) + files = append(files, + seedFile{Path: "usr/local/share/ca-certificates/" + name, Mode: "0644", Content: pemC}, + seedFile{Path: "etc/pki/ca-trust/source/anchors/" + name, Mode: "0644", Content: pemC}) + } + if t := a.agentTLS(base); t != "" { // a root rotation reaches the machine later on + files = append(files, caRefreshFiles(t)...) + } } if sp.NetworkConfig != "" { files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig}) diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index d5effe6..63c92f9 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -32,7 +32,8 @@ func usage() { rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) - rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] | rotate -yes delegate to an ACME CA / replace the root + rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] delegate to an ACME CA + rollcall ca rotate [status | prepare | switch | retire] [-force] replace the root without re-joining racks rollcall version rollcall keygen rollcall list diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index af8e8e6..997d428 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -52,6 +52,9 @@ type RackController struct { Registered time.Time `json:"registered"` Online bool `json:"online"` Machines int `json:"machines"` + // trust (carot.go): what the rack reports at registration + CABundleSHA256 string `json:"ca_bundle_sha256,omitempty"` // SHA-256 of the roots it trusts + ServerCA string `json:"server_ca,omitempty"` // root of its server certificate (agents) } // rackState is everything a rack controller writes to disk. @@ -64,6 +67,15 @@ type rackState struct { Hosts string `json:"hosts"` BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run + Trust *rackTrustState `json:"trust,omitempty"` // the roots to trust (carot.go) +} + +// rackTrustState: the trust bundle signed by the active root, and which root +// signs now (a rack renews certificates from another one). +type rackTrustState struct { + Bundle string `json:"bundle"` + Sig string `json:"sig"` + Active string `json:"active"` // SHA-256 of the active root } type rackJob struct { @@ -91,7 +103,7 @@ type rackResult struct { // ---------------------------------------------------------------- store func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { - rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered, coalesce(server_ca,'') FROM rack_controllers ORDER BY name`) if err != nil { return nil, err } @@ -100,7 +112,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { for rows.Next() { var r RackController var ifs, dif []byte - if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { + if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered, &r.ServerCA); err != nil { return nil, err } _ = json.Unmarshal(ifs, &r.Interfaces) @@ -465,6 +477,9 @@ func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { st := &rackState{Gen: gen} if rc, ok := a.racksSnapshot()[name]; ok { st.Update = a.rackUpdateFor(rc) + if a.ca != nil && rc.Kind == "remote" { + st.Trust = a.trustState() + } } base := a.rackURL(name) app := a.appendFor(base) @@ -856,6 +871,7 @@ func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack st fail(w, 500, err) return } + a.noteRackTrust(r, rack, rc) a.reloadRacks(r.Context()) a.hub.bump() log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces)) diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 2c1c041..f84d3b5 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -476,9 +476,8 @@ func runRack(args []string) error { mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version}) }) - mux.HandleFunc("GET /ca.pem", func(w http.ResponseWriter, req *http.Request) { // agents check it by its fingerprint - http.ServeFile(w, req, cfg.CAFile) - }) + mux.HandleFunc("GET /ca.pem", r.serveCA(false)) // agents check it by its fingerprint + mux.HandleFunc("GET /ca-bundle.pem", r.serveCA(true)) if r.cert != nil && cfg.TLSListen != "off" { go r.serveTLS(logging(mux)) } @@ -534,8 +533,10 @@ func (r *rackd) register() error { r.srvMu.Lock() tlsURL := r.tlsURL r.srvMu.Unlock() + bundle, serverCA := r.trustReport() _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, TLSURL: tlsURL, Version: version, Arch: runtime.GOARCH, - Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil) + Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot, + CABundleSHA256: bundle, ServerCA: serverCA}, nil) return err } @@ -571,6 +572,7 @@ func (r *rackd) loop() { continue } go r.maybeUpdate(st.Update) + go r.maybeTrust(st.Trust) if st.Gen == r.gen { continue } @@ -790,6 +792,12 @@ func (r *rackd) serverCert() (*tls.Certificate, error) { } } } + return r.newServerCert() +} + +// newServerCert asks the region for a server certificate and keeps it. +func (r *rackd) newServerCert() (*tls.Certificate, error) { + p := r.cfg.serverCertPath() key, csr, err := newCSR(rackServerCN + r.cfg.Name) if err != nil { return nil, err diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index e3a2897..537e83d 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -4,7 +4,6 @@ import ( "context" "crypto/subtle" "crypto/tls" - "crypto/x509" "embed" "encoding/json" "errors" @@ -252,6 +251,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs)) mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete)) mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM)) + mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation)) mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS)) mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync)) mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts)) @@ -269,6 +269,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete)) mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP)) mux.HandleFunc("GET /ca.pem", a.handleCAPEM) + mux.HandleFunc("GET /ca-bundle.pem", a.handleCABundle) mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode)) mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew)) @@ -867,8 +868,7 @@ func serve(cfgPath string) error { if err != nil { return fmt.Errorf("region certificate: %w", err) } - pool := x509.NewCertPool() - pool.AddCert(app.ca.Cert) + pool := app.ca.pool() // a rotation's other root too: racks and admins keep working tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}} go func() { diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index 128a0b0..115dc03 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -473,7 +473,7 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) { } caPEM := "" if a.ca != nil { - caPEM = string(a.ca.PEM) + caPEM = string(a.ca.TrustPEM) // during a root rotation both } return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, CA: caPEM, Vars: vars, Server: a.urlFor(m), Now: time.Now().Format(time.RFC3339)}, nil diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index cfa59e9..381d015 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -170,6 +170,9 @@ ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS ca_bundle_sha256 text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS client_ca text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS server_ca text; ALTER TABLE vlans ADD COLUMN IF NOT EXISTS dhcp_mode text NOT NULL DEFAULT ''; CREATE TABLE IF NOT EXISTS join_tokens ( hash text PRIMARY KEY, diff --git a/deploy/rollcall-rack-update.path b/deploy/rollcall-rack-update.path index 1447554..a0146fc 100644 --- a/deploy/rollcall-rack-update.path +++ b/deploy/rollcall-rack-update.path @@ -1,8 +1,9 @@ [Unit] -Description=rollcall: install a rollcall binary the rack controller staged +Description=rollcall: install a rollcall binary or trust bundle the rack controller staged [Path] PathExists=/var/lib/rollcall/update/ready +PathExists=/var/lib/rollcall/update/ca-ready [Install] WantedBy=multi-user.target diff --git a/deploy/rollcall-rack-update.sh b/deploy/rollcall-rack-update.sh index ef370ce..6cef50c 100755 --- a/deploy/rollcall-rack-update.sh +++ b/deploy/rollcall-rack-update.sh @@ -1,17 +1,18 @@ #!/bin/sh -# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path). +# Installs what the rack controller staged (rollcall-rack-update.path): a new +# trust bundle (root rotation) and a new rollcall binary. # # The staging directory belongs to the rack's unprivileged user, so nothing in -# it is trusted: the binary is copied into a directory of root's and installed -# only if the region's signature over it verifies against the CA certificate -# written at join (root's, from rack.json ca_file). Then the running binary is -# kept as .prev, the rack restarted, and the previous binary put back if the -# rack does not register with the region in 60 s. +# it is trusted: files are copied into a directory of root's and installed only +# if the region's signature over them verifies against one of the roots in the +# CA file (root's, from rack.json ca_file; written at join and by this script). +# A bundle replaces the CA file and restarts the rack. A binary is kept as +# .prev, the rack restarted, and the previous binary put back if the rack does +# not register with the region in 60 s. set -u D=/var/lib/rollcall/update BIN=/usr/local/bin/rollcall -[ -f "$D/ready" ] || exit 0 -want=$(head -c 64 "$D/ready" | tr -cd 'A-Za-z0-9._+-'); rm -f "$D/ready" +[ -f "$D/ready" ] || [ -f "$D/ca-ready" ] || exit 0 # root's work directory, on a file system that allows executing (/run is noexec) install -d -m 0700 -o root -g root /var/lib/rollcall-update W=$(mktemp -d /var/lib/rollcall-update/w.XXXXXX) || exit 1 @@ -19,11 +20,51 @@ trap 'rm -rf "$W"' EXIT # last-result is read by the rack (backoff); renamed in, so a link planted there is replaced, not followed result() { echo "$(date -Is) $*" > "$W/result" && mv -fT "$W/result" "$D/last-result"; echo "$*"; } CA=$(jq -r '.ca_file // "/etc/rollcall/rack/ca.pem"' /etc/rollcall/rack.json 2>/dev/null) -[ -n "$CA" ] && [ "$(stat -c %u "$CA" 2>/dev/null)" = 0 ] || { result "$want: no root-owned CA certificate ($CA), not installed"; exit 1; } +[ -n "$CA" ] && [ "$(stat -c %u "$CA" 2>/dev/null)" = 0 ] || { result "no root-owned CA certificate ($CA), nothing installed"; exit 1; } +# split PEM certificates of $1 into $2.1, $2.2, ... +splitpem() { awk -v p="$2" '/-----BEGIN CERTIFICATE-----/{n++; f=p "." n} f{print > f} /-----END CERTIFICATE-----/{close(f); f=""}' "$1"; } +# signed FILE SIG: the signature verifies against one of the roots in $CA +signed() { + rm -f "$W"/root.*; splitpem "$CA" "$W/root" + for c in "$W"/root.*; do + [ -f "$c" ] || continue + openssl x509 -in "$c" -pubkey -noout > "$W/pub" 2>/dev/null && + openssl dgst -sha256 -verify "$W/pub" -signature "$2" "$1" >/dev/null 2>&1 && return 0 + done + return 1 +} + +if [ -f "$D/ca-ready" ]; then + rm -f "$D/ca-ready" + if cp -- "$D/ca-bundle.pem" "$W/bundle" && cp -- "$D/ca-bundle.sig" "$W/bundle.sig"; then + if ! signed "$W/bundle" "$W/bundle.sig"; then + result "trust bundle: signature does not verify against $CA, not installed" + else + # only CA certificates, rewritten from what was parsed + rm -f "$W"/new.*; splitpem "$W/bundle" "$W/new"; : > "$W/ca.new"; n=0; ok=1 + for c in "$W"/new.*; do + [ -f "$c" ] || continue + openssl x509 -in "$c" -noout -text 2>/dev/null | grep -q "CA:TRUE" || ok= + openssl x509 -in "$c" >> "$W/ca.new" 2>/dev/null || ok= + n=$((n+1)) + done + if [ -z "$ok" ] || [ $n -eq 0 ]; then + result "trust bundle: not CA certificates only, not installed" + else + cp -p "$CA" "$CA.prev" + install -m 0644 -o root -g root "$W/ca.new" "$CA.tmp" && mv -f "$CA.tmp" "$CA" + rm -f "$D/ca-bundle.pem" "$D/ca-bundle.sig" + systemctl restart rollcall-rack.service + result "trust bundle installed ($n roots)" + fi + fi + fi +fi + +[ -f "$D/ready" ] || exit 0 +want=$(head -c 64 "$D/ready" | tr -cd 'A-Za-z0-9._+-'); rm -f "$D/ready" cp -- "$D/rollcall.new" "$W/bin" && cp -- "$D/rollcall.new.sig" "$W/sig" || { result "$want: staged files missing, not installed"; exit 1; } -openssl x509 -in "$CA" -pubkey -noout > "$W/pub" 2>/dev/null && - openssl dgst -sha256 -verify "$W/pub" -signature "$W/sig" "$W/bin" >/dev/null 2>&1 || - { result "$want: signature does not verify against $CA, not installed"; exit 1; } +signed "$W/bin" "$W/sig" || { result "$want: signature does not verify against $CA, not installed"; exit 1; } chmod 0755 "$W/bin" got=$("$W/bin" version 2>/dev/null | awk '{print $2}') [ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; }