The CA directory holds the active root and, during a rotation, the next or the previous one; everything that trusts the region gets the trust bundle (all of them). Rack controllers receive it in their state, signed by the active root; the root update unit installs it only when the signature verifies against the roots it already trusts, and binary signatures verify against any of them. After switch racks renew their client and server certificates from the new root by themselves and report what they run on; switch and retire wait for every rack (-force). /ca.pem stays one certificate, the root of that server's certificate, and the PXE command line names that root, so agents in older images keep working; /ca-bundle.pem is the bundle. Deployed machines get one file per root and a daily timer that refreshes them from their rack over HTTPS. Refs #12 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
674 lines
21 KiB
Go
674 lines
21 KiB
Go
package main
|
|
|
|
// Root rotation without re-joining. The CA directory holds the active root
|
|
// (ca.pem, ca-key.pem) and, during a rotation, the next one (ca-next.pem,
|
|
// ca-next-key.pem) or the previous one (ca-prev.pem). Everything that trusts
|
|
// the region gets the trust bundle (all of them, active first):
|
|
//
|
|
// prepare a new root is made; the old one still signs. Rack controllers
|
|
// install the bundle (signed by the active root, installed by their
|
|
// root unit), machines deployed from now on trust both.
|
|
// switch the new root signs: the region's certificate, rack and admin
|
|
// certificates from now on. Racks renew their client and server
|
|
// certificates by themselves; the old root stays trusted.
|
|
// retire the old root leaves the bundle once every rack runs on the new one.
|
|
//
|
|
// switch and retire refuse while a rack is not ready (-force overrides).
|
|
// /ca.pem stays one certificate (the root of the server's own certificate:
|
|
// agents in older images read one); /ca-bundle.pem is the bundle.
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/x509"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/pem"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// loadTrust adds the next and previous roots of a rotation to ca.Trust.
|
|
func (ca *CA) loadTrust(dir string) error {
|
|
ca.Trust = []*x509.Certificate{ca.Cert}
|
|
ca.TrustPEM = append([]byte{}, ca.PEM...)
|
|
for _, f := range []string{"ca-next.pem", "ca-prev.pem"} {
|
|
b, err := os.ReadFile(filepath.Join(dir, f))
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c, err := parseCertPEM(b)
|
|
if err != nil || !c.IsCA {
|
|
return fmt.Errorf("%s: not a CA certificate", f)
|
|
}
|
|
ca.Trust = append(ca.Trust, c)
|
|
ca.TrustPEM = append(ca.TrustPEM, pemEncode("CERTIFICATE", c.Raw)...)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func fpHex(c *x509.Certificate) string {
|
|
sum := certFingerprint(c)
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func (ca *CA) pool() *x509.CertPool {
|
|
p := x509.NewCertPool()
|
|
for _, c := range ca.Trust {
|
|
p.AddCert(c)
|
|
}
|
|
return p
|
|
}
|
|
|
|
// signBytes is the active root's signature over data's SHA-256 (base64 DER),
|
|
// as for rack binaries: openssl dgst -sha256 -verify checks it.
|
|
func (ca *CA) signBytes(data []byte) string {
|
|
sum := sha256.Sum256(data)
|
|
der, err := ecdsa.SignASN1(rand.Reader, ca.Key, sum[:])
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return base64.StdEncoding.EncodeToString(der)
|
|
}
|
|
|
|
// rootOf is the root among trust that signed c, nil when none did.
|
|
func rootOf(c *x509.Certificate, trust []*x509.Certificate) *x509.Certificate {
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
for _, r := range trust {
|
|
if bytes.Equal(c.RawIssuer, r.RawSubject) && c.CheckSignatureFrom(r) == nil {
|
|
return r
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// verifyBundle: a bundle the region sent, signed by one of the roots trusted
|
|
// now, holding CA certificates only.
|
|
func verifyBundle(bundle []byte, sig string, trust []*x509.Certificate) ([]*x509.Certificate, error) {
|
|
der, err := base64.StdEncoding.DecodeString(sig)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("bundle signature: %w", err)
|
|
}
|
|
sum := sha256.Sum256(bundle)
|
|
ok := false
|
|
for _, r := range trust {
|
|
if pub, isEC := r.PublicKey.(*ecdsa.PublicKey); isEC && ecdsa.VerifyASN1(pub, sum[:], der) {
|
|
ok = true
|
|
}
|
|
}
|
|
if !ok {
|
|
return nil, errors.New("the bundle is not signed by a root trusted here")
|
|
}
|
|
certs, err := parsePEMCerts(bundle)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, c := range certs {
|
|
if !c.IsCA {
|
|
return nil, fmt.Errorf("bundle: %q is not a CA certificate", c.Subject.CommonName)
|
|
}
|
|
}
|
|
return certs, nil
|
|
}
|
|
|
|
func parsePEMCerts(b []byte) ([]*x509.Certificate, error) {
|
|
var out []*x509.Certificate
|
|
for {
|
|
var blk *pem.Block
|
|
blk, b = pem.Decode(b)
|
|
if blk == nil {
|
|
break
|
|
}
|
|
if blk.Type != "CERTIFICATE" {
|
|
return nil, fmt.Errorf("unexpected PEM block %s", blk.Type)
|
|
}
|
|
c, err := x509.ParseCertificate(blk.Bytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, c)
|
|
}
|
|
if len(out) == 0 {
|
|
return nil, errors.New("no certificate")
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------- store
|
|
|
|
// SetRackTrust records what a rack runs on: the hash of its trust bundle and
|
|
// the roots of its client and server certificates.
|
|
func (s *Store) SetRackTrust(ctx context.Context, name, bundleSHA, clientCA, serverCA string) error {
|
|
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET ca_bundle_sha256=nullif($2,''), client_ca=nullif($3,''),
|
|
server_ca=coalesce(nullif($4,''), server_ca) WHERE name=$1`, name, bundleSHA, clientCA, serverCA)
|
|
return err
|
|
}
|
|
|
|
type rackTrust struct {
|
|
Name string `json:"name"`
|
|
Bundle string `json:"ca_bundle_sha256"`
|
|
ClientCA string `json:"client_ca"`
|
|
ServerCA string `json:"server_ca"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
}
|
|
|
|
func (s *Store) RackTrust(ctx context.Context) ([]rackTrust, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(ca_bundle_sha256,''), coalesce(client_ca,''), coalesce(server_ca,''), last_seen
|
|
FROM rack_controllers WHERE kind='remote' ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := []rackTrust{}
|
|
for rows.Next() {
|
|
var t rackTrust
|
|
if err := rows.Scan(&t.Name, &t.Bundle, &t.ClientCA, &t.ServerCA, &t.LastSeen); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, t)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// ---------------------------------------------------------------- region HTTP
|
|
|
|
func (a *App) handleCABundle(w http.ResponseWriter, r *http.Request) {
|
|
if a.ca == nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/x-pem-file")
|
|
_, _ = w.Write(a.ca.TrustPEM)
|
|
}
|
|
|
|
type rotationStatus struct {
|
|
Phase string `json:"phase"` // "" | prepared | switched
|
|
Active string `json:"active"`
|
|
Next string `json:"next,omitempty"`
|
|
Prev string `json:"prev,omitempty"`
|
|
Bundle string `json:"bundle_sha256"`
|
|
Racks []rackTrust `json:"racks"`
|
|
// what blocks the next step
|
|
NotReady []string `json:"not_ready"`
|
|
}
|
|
|
|
func (a *App) rotation(ctx context.Context) (rotationStatus, error) {
|
|
st := rotationStatus{Active: fpHex(a.ca.Cert), NotReady: []string{}}
|
|
sum := sha256.Sum256(a.ca.TrustPEM)
|
|
st.Bundle = hex.EncodeToString(sum[:])
|
|
if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-next.pem")); err == nil {
|
|
if c, err := parseCertPEM(b); err == nil {
|
|
st.Phase, st.Next = "prepared", fpHex(c)
|
|
}
|
|
}
|
|
if b, err := os.ReadFile(filepath.Join(a.cfg.CADir, "ca-prev.pem")); err == nil {
|
|
if c, err := parseCertPEM(b); err == nil {
|
|
st.Phase, st.Prev = "switched", fpHex(c)
|
|
}
|
|
}
|
|
racks, err := a.store.RackTrust(ctx)
|
|
if err != nil {
|
|
return st, err
|
|
}
|
|
st.Racks = racks
|
|
for _, r := range racks {
|
|
switch st.Phase {
|
|
case "prepared": // switch needs the new root at every rack
|
|
if r.Bundle != st.Bundle {
|
|
st.NotReady = append(st.NotReady, r.Name+": has not installed the bundle with the new root")
|
|
}
|
|
case "switched": // retire needs every rack on certificates from the new root
|
|
if r.ClientCA != st.Active || r.ServerCA != "" && r.ServerCA != st.Active {
|
|
st.NotReady = append(st.NotReady, r.Name+": still has a certificate from the old root")
|
|
}
|
|
}
|
|
}
|
|
return st, nil
|
|
}
|
|
|
|
func (a *App) handleCARotation(w http.ResponseWriter, r *http.Request) {
|
|
if a.ca == nil {
|
|
fail(w, 409, errors.New("no CA"))
|
|
return
|
|
}
|
|
st, err := a.rotation(r.Context())
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, st)
|
|
}
|
|
|
|
// noteRackTrust records, at registration, the root of the client certificate
|
|
// the rack connected with and what it says about its bundle and server certificate.
|
|
func (a *App) noteRackTrust(r *http.Request, rack string, rc RackController) {
|
|
if a.ca == nil || r.TLS == nil {
|
|
return
|
|
}
|
|
var client string
|
|
for _, ch := range r.TLS.VerifiedChains {
|
|
if len(ch) == 2 {
|
|
client = fpHex(ch[1])
|
|
}
|
|
}
|
|
_ = a.store.SetRackTrust(r.Context(), rack, rc.CABundleSHA256, client, rc.ServerCA)
|
|
}
|
|
|
|
// serverCAFor is the root an agent behind this rack must accept: that of the
|
|
// rack's server certificate (it serves that one as /ca.pem).
|
|
func (a *App) serverCAFor(base string) *x509.Certificate {
|
|
if base != a.cfg.ServerURL {
|
|
for _, rc := range a.racksSnapshot() {
|
|
if strings.TrimRight(rc.URL, "/") == base && rc.ServerCA != "" {
|
|
for _, c := range a.ca.Trust {
|
|
if fpHex(c) == rc.ServerCA {
|
|
return c
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return a.ca.Cert
|
|
}
|
|
|
|
var trustSig struct {
|
|
sync.Mutex
|
|
bundle, sig string
|
|
}
|
|
|
|
// trustState is signed once per bundle: the rack state stays the same between polls.
|
|
func (a *App) trustState() *rackTrustState {
|
|
trustSig.Lock()
|
|
defer trustSig.Unlock()
|
|
if trustSig.bundle != string(a.ca.TrustPEM) {
|
|
trustSig.bundle, trustSig.sig = string(a.ca.TrustPEM), a.ca.signBytes(a.ca.TrustPEM)
|
|
}
|
|
return &rackTrustState{Bundle: trustSig.bundle, Sig: trustSig.sig, Active: fpHex(a.ca.Cert)}
|
|
}
|
|
|
|
// caRefreshFiles keep a deployed machine's roots current: the bundle comes
|
|
// over HTTPS from the rack, checked against the roots the machine trusts now,
|
|
// so a rotation reaches it during prepare (before the new root signs) and the
|
|
// old root leaves it after retire.
|
|
func caRefreshFiles(tlsBase string) []seedFile {
|
|
script := `#!/bin/sh
|
|
# rollcall: keep the installation's roots current (written at deploy time)
|
|
set -eu
|
|
URL=` + shQuote(tlsBase+"/ca-bundle.pem") + `
|
|
T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
|
|
curl -fsS --max-time 30 -o "$T/bundle" "$URL" || exit 0 # unreachable: the timer tries again
|
|
awk -v d="$T" '/-----BEGIN CERTIFICATE-----/{n++} n{print > (d "/c" n)} /-----END CERTIFICATE-----/{close(d "/c" n)}' "$T/bundle"
|
|
[ -f "$T/c1" ] || exit 0
|
|
changed=
|
|
for dir in /usr/local/share/ca-certificates /etc/pki/ca-trust/source/anchors; do
|
|
[ -d "$dir" ] || continue
|
|
i=0; mkdir -p "$T/want"; rm -f "$T"/want/*
|
|
for c in "$T"/c[0-9]*; do
|
|
i=$((i+1)); n=rollcall-ca.crt; [ $i -gt 1 ] && n=rollcall-ca-$i.crt
|
|
cp "$c" "$T/want/$n"
|
|
done
|
|
if [ "$(cat "$dir"/rollcall-ca*.crt 2>/dev/null)" != "$(cat "$T"/want/*)" ]; then
|
|
rm -f "$dir"/rollcall-ca*.crt
|
|
for f in "$T"/want/*; do install -m 0644 "$f" "$dir/$(basename "$f")"; done
|
|
changed=1
|
|
fi
|
|
done
|
|
[ -n "$changed" ] || exit 0
|
|
if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi
|
|
echo "rollcall: trusted roots updated from $URL"
|
|
`
|
|
return []seedFile{
|
|
{Path: "usr/local/sbin/rollcall-ca-refresh", Mode: "0755", Content: script},
|
|
{Path: "etc/systemd/system/rollcall-ca-refresh.service", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/rollcall-ca-refresh\n"},
|
|
{Path: "etc/systemd/system/rollcall-ca-refresh.timer", Mode: "0644", Content: "[Unit]\nDescription=rollcall: refresh the installation's trusted roots daily\n\n[Timer]\nOnBootSec=10min\nOnUnitActiveSec=1d\nRandomizedDelaySec=1h\n\n[Install]\nWantedBy=timers.target\n"},
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- CLI
|
|
|
|
// rollcall ca rotate [status | prepare | switch | retire] [-force]
|
|
func caRotateCmd(cfg *Config, args []string) error {
|
|
sub := "status"
|
|
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
sub, args = args[0], args[1:]
|
|
}
|
|
fs := newFlags("ca rotate " + sub)
|
|
force := fs.Bool("force", false, "go on although rack controllers are not ready (they then join again)")
|
|
_ = fs.Parse(args)
|
|
dir := cfg.CADir
|
|
have := func(f string) bool { _, err := os.Stat(filepath.Join(dir, f)); return err == nil }
|
|
status := func() (*rotationStatus, error) {
|
|
c, err := newClient()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var st rotationStatus
|
|
if err := c.do("GET", "/api/v1/ca/rotation", nil, &st); err != nil {
|
|
return nil, err
|
|
}
|
|
return &st, nil
|
|
}
|
|
ready := func() error {
|
|
st, err := status()
|
|
if err != nil {
|
|
if *force {
|
|
fmt.Fprintf(os.Stderr, "cannot ask the region (%v); -force: going on\n", err)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("cannot ask the region whether rack controllers are ready: %w (-force to go on)", err)
|
|
}
|
|
if len(st.NotReady) > 0 && !*force {
|
|
return fmt.Errorf("not ready:\n %s\nwait for them (they pick it up within minutes when online) or -force (they then join again)", strings.Join(st.NotReady, "\n "))
|
|
}
|
|
return nil
|
|
}
|
|
restart := "next: restart rollcall (systemctl restart rollcall); `rollcall ca rotate` shows the progress"
|
|
switch sub {
|
|
case "status":
|
|
st, err := status()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
phase := map[string]string{"": "no rotation", "prepared": "prepared (next: switch)", "switched": "switched (next: retire)"}[st.Phase]
|
|
fmt.Printf("phase %s\nactive %s\n", phase, fpString(mustFP(st.Active)))
|
|
if st.Next != "" {
|
|
fmt.Printf("next %s\n", fpString(mustFP(st.Next)))
|
|
}
|
|
if st.Prev != "" {
|
|
fmt.Printf("prev %s (still trusted)\n", fpString(mustFP(st.Prev)))
|
|
}
|
|
short := func(fp string) string {
|
|
switch {
|
|
case fp == "":
|
|
return "-"
|
|
case fp == st.Active:
|
|
return "active"
|
|
case fp == st.Next:
|
|
return "next"
|
|
case fp == st.Prev:
|
|
return "prev"
|
|
}
|
|
return fp[:12]
|
|
}
|
|
fmt.Printf("\n%-16s %-8s %-8s %-8s %s\n", "RACK", "BUNDLE", "CLIENT", "SERVER", "LAST SEEN")
|
|
for _, r := range st.Racks {
|
|
b := "old"
|
|
if r.Bundle == st.Bundle {
|
|
b = "current"
|
|
}
|
|
seen := "-"
|
|
if r.LastSeen != nil {
|
|
seen = r.LastSeen.Local().Format("2006-01-02 15:04")
|
|
}
|
|
fmt.Printf("%-16s %-8s %-8s %-8s %s\n", r.Name, b, short(r.ClientCA), short(r.ServerCA), seen)
|
|
}
|
|
for _, n := range st.NotReady {
|
|
fmt.Println("waiting:", n)
|
|
}
|
|
return nil
|
|
case "prepare":
|
|
if have("ca-next.pem") || have("ca-prev.pem") {
|
|
return errors.New("a rotation is under way (rollcall ca rotate status)")
|
|
}
|
|
next, err := newCA("rollcall CA " + time.Now().Format("2006-01-02"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeCA(dir, "ca-next", next); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("next root %s\nthe active root still signs; rack controllers and newly deployed machines get both\n%s; then `rollcall ca rotate switch`\n", next.Fingerprint(), restart)
|
|
return nil
|
|
case "switch":
|
|
if !have("ca-next.pem") {
|
|
return errors.New("no prepared root (rollcall ca rotate prepare)")
|
|
}
|
|
if err := ready(); err != nil {
|
|
return err
|
|
}
|
|
bak := filepath.Join(dir, "rotated-"+time.Now().Format("20060102-150405"))
|
|
if err := os.Mkdir(bak, 0o700); err != nil {
|
|
return err
|
|
}
|
|
steps := [][2]string{{"ca.pem", "ca-prev.pem"}, {"ca-key.pem", filepath.Join(filepath.Base(bak), "ca-key.pem")},
|
|
{"ca-next-key.pem", "ca-key.pem"}, {"ca-next.pem", "ca.pem"}}
|
|
for _, s := range steps {
|
|
if err := copyKeepOwner(filepath.Join(dir, s[0]), filepath.Join(dir, s[1])); err != nil {
|
|
return fmt.Errorf("%s -> %s: %w (the CA directory may be half switched; the old files are in %s)", s[0], s[1], err, bak)
|
|
}
|
|
}
|
|
for _, f := range []string{"ca-next.pem", "ca-next-key.pem"} {
|
|
_ = os.Remove(filepath.Join(dir, f))
|
|
}
|
|
fmt.Printf("the new root signs now; the old one stays trusted (its key: %s)\nrack controllers renew their certificates by themselves; admin certificates and intermediates from the old root work until retire: issue them again\n%s; then `rollcall ca rotate retire`\n", bak, restart)
|
|
return nil
|
|
case "retire":
|
|
if !have("ca-prev.pem") {
|
|
return errors.New("no previous root to retire")
|
|
}
|
|
if err := ready(); err != nil {
|
|
return err
|
|
}
|
|
bak := filepath.Join(dir, "retired-"+time.Now().Format("20060102-150405")+".pem")
|
|
if err := os.Rename(filepath.Join(dir, "ca-prev.pem"), bak); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("the old root is no longer trusted (kept in %s)\n%s\n", bak, restart)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("ca rotate %s: unknown (status, prepare, switch, retire)", sub)
|
|
}
|
|
|
|
func mustFP(h string) [32]byte {
|
|
var out [32]byte
|
|
b, _ := hex.DecodeString(h)
|
|
copy(out[:], b)
|
|
return out
|
|
}
|
|
|
|
// writeCA writes a root as <name>.pem / <name>-key.pem, owned like ca.pem.
|
|
func writeCA(dir, name string, ca *CA) error {
|
|
kder, err := x509.MarshalECPrivateKey(ca.Key)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kp, cp := filepath.Join(dir, name+"-key.pem"), filepath.Join(dir, name+".pem")
|
|
if err := os.WriteFile(kp, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(cp, ca.PEM, 0o644); err != nil {
|
|
return err
|
|
}
|
|
if own, ok := fileOwner(filepath.Join(dir, "ca.pem")); ok {
|
|
_ = os.Chown(kp, own[0], own[1])
|
|
_ = os.Chown(cp, own[0], own[1])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// copyKeepOwner copies src over dst (atomically) with src's mode and owner.
|
|
func copyKeepOwner(src, dst string) error {
|
|
b, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
st, err := os.Stat(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomic(dst, b, st.Mode().Perm()); err != nil {
|
|
return err
|
|
}
|
|
if own, ok := fileOwner(src); ok {
|
|
_ = os.Chown(dst, own[0], own[1])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// shQuote is s as one single-quoted shell word.
|
|
func shQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
|
|
|
|
// ---------------------------------------------------------------- rack controller
|
|
|
|
// rackTrust is what the rack trusts now (its ca_file, written at join and by
|
|
// the root update unit).
|
|
func (r *rackd) rackTrust() ([]byte, []*x509.Certificate) {
|
|
b, err := os.ReadFile(r.cfg.CAFile)
|
|
if err != nil {
|
|
return nil, nil
|
|
}
|
|
certs, _ := parsePEMCerts(b)
|
|
return b, certs
|
|
}
|
|
|
|
// trustReport: the hash of the rack's roots and the root of its server certificate.
|
|
func (r *rackd) trustReport() (bundleSHA, serverCA string) {
|
|
b, trust := r.rackTrust()
|
|
if b == nil {
|
|
return "", ""
|
|
}
|
|
sum := sha256.Sum256(b)
|
|
r.srvMu.Lock()
|
|
c := r.srvCert
|
|
r.srvMu.Unlock()
|
|
if c != nil {
|
|
if root := rootOf(c.Leaf, trust); root != nil {
|
|
serverCA = fpHex(root)
|
|
}
|
|
}
|
|
return hex.EncodeToString(sum[:]), serverCA
|
|
}
|
|
|
|
// serveCA answers /ca.pem with the root of the server certificate (agents
|
|
// pin it by fingerprint and read one certificate) and /ca-bundle.pem with all.
|
|
func (r *rackd) serveCA(bundle bool) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, req *http.Request) {
|
|
b, trust := r.rackTrust()
|
|
if b == nil {
|
|
http.Error(w, "no CA", 404)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/x-pem-file")
|
|
if bundle || len(trust) < 2 {
|
|
_, _ = w.Write(b)
|
|
return
|
|
}
|
|
r.srvMu.Lock()
|
|
c := r.srvCert
|
|
r.srvMu.Unlock()
|
|
root := trust[0]
|
|
if c != nil {
|
|
if o := rootOf(c.Leaf, trust); o != nil {
|
|
root = o
|
|
}
|
|
}
|
|
_, _ = w.Write(pemEncode("CERTIFICATE", root.Raw))
|
|
}
|
|
}
|
|
|
|
var rackTrustMu sync.Mutex
|
|
var rackTrustTried = map[string]time.Time{}
|
|
|
|
// maybeTrust follows the region's roots: a new bundle is staged for the root
|
|
// unit, and certificates from a root that no longer signs are renewed.
|
|
func (r *rackd) maybeTrust(t *rackTrustState) {
|
|
if t == nil || r.cert == nil {
|
|
return
|
|
}
|
|
rackTrustMu.Lock()
|
|
defer rackTrustMu.Unlock()
|
|
cur, trust := r.rackTrust()
|
|
if t.Bundle != "" && t.Bundle != string(cur) {
|
|
sum := sha256.Sum256([]byte(t.Bundle))
|
|
key := "bundle " + hex.EncodeToString(sum[:])
|
|
if time.Since(rackTrustTried[key]) > time.Hour {
|
|
rackTrustTried[key] = time.Now()
|
|
if err := r.stageBundle(t, trust); err != nil {
|
|
log.Printf("trust bundle: %v", err)
|
|
} else {
|
|
log.Printf("new trust bundle staged in %s; rollcall-rack-update installs it", r.updateDir())
|
|
}
|
|
}
|
|
}
|
|
var active *x509.Certificate
|
|
for _, c := range trust {
|
|
if fpHex(c) == t.Active {
|
|
active = c
|
|
}
|
|
}
|
|
if active == nil {
|
|
return // the signing root is not trusted here yet: the bundle comes first
|
|
}
|
|
renewed := false
|
|
r.certMu.Lock()
|
|
leaf := r.cert.Leaf
|
|
r.certMu.Unlock()
|
|
if rootOf(leaf, trust) != active && time.Since(rackTrustTried["client"]) > 10*time.Minute {
|
|
rackTrustTried["client"] = time.Now()
|
|
if err := r.renew(); err != nil {
|
|
log.Printf("certificate from the new root: %v", err)
|
|
} else {
|
|
renewed = true
|
|
}
|
|
}
|
|
r.srvMu.Lock()
|
|
srv := r.srvCert
|
|
r.srvMu.Unlock()
|
|
if srv != nil && rootOf(srv.Leaf, trust) != active && time.Since(rackTrustTried["server"]) > 10*time.Minute {
|
|
rackTrustTried["server"] = time.Now()
|
|
if c, err := r.newServerCert(); err != nil {
|
|
log.Printf("server certificate from the new root: %v", err)
|
|
} else {
|
|
r.srvMu.Lock()
|
|
r.srvCert = c
|
|
r.srvMu.Unlock()
|
|
renewed = true
|
|
}
|
|
}
|
|
if renewed {
|
|
for _, hc := range []*http.Client{r.hc, r.fast, r.big} { // new connections present the new certificate
|
|
if tr, ok := hc.Transport.(*http.Transport); ok {
|
|
tr.CloseIdleConnections()
|
|
}
|
|
}
|
|
if err := r.register(); err != nil {
|
|
log.Printf("register after renewal: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// stageBundle leaves a verified bundle and its signature for the root unit,
|
|
// which checks the signature again against its own copy of the roots.
|
|
func (r *rackd) stageBundle(t *rackTrustState, trust []*x509.Certificate) error {
|
|
if _, err := verifyBundle([]byte(t.Bundle), t.Sig, trust); err != nil {
|
|
return err
|
|
}
|
|
sig, _ := base64.StdEncoding.DecodeString(t.Sig)
|
|
dir := r.updateDir()
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomic(filepath.Join(dir, "ca-bundle.pem"), []byte(t.Bundle), 0o644); err != nil {
|
|
return err
|
|
}
|
|
if err := writeAtomic(filepath.Join(dir, "ca-bundle.sig"), sig, 0o644); err != nil {
|
|
return err
|
|
}
|
|
return writeAtomic(filepath.Join(dir, "ca-ready"), []byte("1\n"), 0o644)
|
|
}
|