diff --git a/agent/rollcall-agent.py b/agent/rollcall-agent.py index fc2b084..5d50961 100644 --- a/agent/rollcall-agent.py +++ b/agent/rollcall-agent.py @@ -13,11 +13,13 @@ Kernel cmdline options: rollcall.mode=deploy rollcall.job=.. run rollcall-deploy.py instead (OS deployment / release job) """ import glob +import hashlib import json import os import re import secrets import socket +import ssl import string import subprocess import sys @@ -399,6 +401,36 @@ def bmc_inband(user_name): AGENT_TOKEN = "" +def tls_setup(opts, server): + """rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and + accepted only by that fingerprint; from then on everything goes over HTTPS + with it. When TLS was asked for and does not work, nothing falls back to HTTP.""" + url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower() + if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64: + return server, None, None + pem = None + for i in range(10): + try: + with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r: + pem = r.read().decode() + break + except (urllib.error.URLError, OSError) as e: + log(f"ca.pem: {e}, retry {i + 1}/10") + time.sleep(3) + if pem is None: + raise RuntimeError("cannot fetch the CA certificate") + if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want: + raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server") + path = "/run/rollcall-ca.pem" + with open(path, "w") as f: + f.write(pem) + log(f"talking to {url} over HTTPS (CA {want[:16]}...)") + return url.rstrip("/"), ssl.create_default_context(cafile=path), path + + +TLS = None # ssl context once tls_setup ran + + def agent_headers(): h = {"Content-Type": "application/json"} if AGENT_TOKEN: @@ -409,7 +441,7 @@ def agent_headers(): def bmc_hint(server, macs): try: req = urllib.request.Request(f"{server}/api/v1/bmc-hint?macs={','.join(macs)}", headers=agent_headers()) - with urllib.request.urlopen(req, timeout=10) as r: + with urllib.request.urlopen(req, timeout=10, context=TLS) as r: h = json.load(r) except (urllib.error.URLError, json.JSONDecodeError, OSError): return None @@ -439,7 +471,7 @@ def post(server, payload, tries=30): req = urllib.request.Request(f"{server}/api/v1/commission", data=data, headers=agent_headers(), method="POST") for i in range(tries): try: - with urllib.request.urlopen(req, timeout=30) as r: + with urllib.request.urlopen(req, timeout=30, context=TLS) as r: return json.load(r) except (urllib.error.URLError, OSError) as e: log(f"post failed ({e}), retry {i + 1}/{tries}") @@ -457,7 +489,8 @@ def main(): log("rollcall.server= missing on kernel cmdline; nothing to do") return 2 server = server.rstrip("/") - global AGENT_TOKEN + global AGENT_TOKEN, TLS + server, TLS, _ = tls_setup(opts, server) AGENT_TOKEN = opts.get("token") if isinstance(opts.get("token"), str) else "" user_name = opts.get("bmcuser") or "commissioner" started = time.time() diff --git a/agent/rollcall-deploy.py b/agent/rollcall-deploy.py index 11b2353..dcde711 100644 --- a/agent/rollcall-deploy.py +++ b/agent/rollcall-deploy.py @@ -16,6 +16,7 @@ import hashlib import json import os import re +import ssl import subprocess import sys import threading @@ -62,6 +63,35 @@ def cmdline(): # --------------------------------------------------------------------------- server +TLS, CAFILE = None, None + +def tls_setup(opts, server): + """rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and + accepted only by that fingerprint; from then on everything goes over HTTPS + with it. When TLS was asked for and does not work, nothing falls back to HTTP.""" + url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower() + if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64: + return server, None, None + pem = None + for i in range(10): + try: + with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r: + pem = r.read().decode() + break + except (urllib.error.URLError, OSError) as e: + log(f"ca.pem: {e}, retry {i + 1}/10") + time.sleep(3) + if pem is None: + raise RuntimeError("cannot fetch the CA certificate") + if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want: + raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server") + path = "/run/rollcall-ca.pem" + with open(path, "w") as f: + f.write(pem) + log(f"talking to {url} over HTTPS (CA {want[:16]}...)") + return url.rstrip("/"), ssl.create_default_context(cafile=path), path + + def api(method, path, body=None, tries=20): data = json.dumps(body).encode() if body is not None else None req = urllib.request.Request(SERVER + path, data=data, method=method, @@ -69,7 +99,7 @@ def api(method, path, body=None, tries=20): last = None for i in range(tries): try: - with urllib.request.urlopen(req, timeout=30) as r: + with urllib.request.urlopen(req, timeout=30, context=TLS) as r: return json.load(r) except urllib.error.HTTPError as e: if 400 <= e.code < 500: @@ -196,7 +226,8 @@ def write_image(spec, disk, prog): ["bash", "-c", 'set -o pipefail; pv -n -f -i 2 -s "$1" 2>"$2" | zstd -dc -T0 ' '| dd of="$3" bs=4M iflag=fullblock oflag=direct conv=fsync status=none', "write", str(size), pvf, disk], stdin=subprocess.PIPE, stderr=subprocess.PIPE) - curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused", "--", url], + cacert = ["--cacert", CAFILE] if CAFILE and url.startswith("https://") else [] + curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused"] + cacert + ["--", url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) log(f"writing {url} -> {disk}") h, pct, shown = hashlib.sha256(), 0, 0.0 @@ -370,6 +401,8 @@ def main(): global SERVER, TOKEN opts = cmdline() SERVER = (opts.get("server") or "").rstrip("/") + global TLS, CAFILE + SERVER, TLS, CAFILE = tls_setup(opts, SERVER) TOKEN = opts.get("job") if isinstance(opts.get("job"), str) else "" if not SERVER or not TOKEN: log("rollcall.server= and rollcall.job= required") diff --git a/cmd/rollcall/agenttls.go b/cmd/rollcall/agenttls.go new file mode 100644 index 0000000..0d9ab57 --- /dev/null +++ b/cmd/rollcall/agenttls.go @@ -0,0 +1,171 @@ +package main + +// TLS for the agents (commissioning, deploy, stateless nodes). Rack +// controllers serve the node API and images over HTTPS too, with a server +// certificate the region issues for the rack's own address only (never one +// of the region's). The pxelinux command line names the HTTPS URL and the +// CA's fingerprint (rollcall.tls=, rollcall.ca=); an agent fetches the CA +// certificate, accepts it only by that fingerprint, and then talks HTTPS +// with it. The embedded rack uses the region's own certificate. +// +// This keeps what agents send and receive (BMC passwords, seed files) from +// anyone who can only listen, and from an HTTP server posing as the rack. +// PXE itself (TFTP, the kernel, the initrd) stays unauthenticated: an +// attacker who can rewrite the boot files on the wire is not stopped here; +// that takes UEFI Secure Boot with signed images. + +import ( + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/url" + "strings" +) + +// rackServerCN is the subject of a rack controller's server certificate; +// regionLeaf and the CLI refuse it as the region. +const rackServerCN = "rack-server:" + +// handleRackServerCert issues a rack's server certificate for the host of its +// registered URL: {"csr"} -> {"cert"}. +func (a *App) handleRackServerCert(w http.ResponseWriter, r *http.Request, rack string) { + if a.peerRack(r) != rack || a.ca == nil { + fail(w, 401, errors.New("a server certificate needs the rack's own certificate")) + return + } + rc, ok := a.racksSnapshot()[rack] + if !ok { + fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack)) + return + } + host, err := rackHost(rc.URL) + if err != nil { + fail(w, 400, err) + return + } + if a.isRegionName(host) { + fail(w, 403, fmt.Errorf("%s is the region's own address: no rack certificate for it", host)) + return + } + var req enrollRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil { + fail(w, 400, err) + return + } + csr, err := parseCSR(req.CSR) + if err != nil { + fail(w, 400, err) + return + } + certPEM, cert, err := a.ca.sign(csr.PublicKey, rackServerCN+rack, []string{host}, x509.ExtKeyUsageServerAuth) + if err != nil { + fail(w, 500, err) + return + } + log.Printf("rack controller %s: server certificate for %s until %s", rack, host, cert.NotAfter.Format("2006-01-02")) + writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.PEM)}) +} + +// rackHost is the host part of a rack URL (an address or a name). +func rackHost(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil || u.Hostname() == "" { + return "", fmt.Errorf("rack URL %q has no host", raw) + } + return u.Hostname(), nil +} + +// isRegionName: one of the names or addresses the region's own certificate covers. +func (a *App) isRegionName(host string) bool { + h := strings.ToLower(strings.Trim(host, "[]")) + if ad, ok := parseAddr(h); ok { + h = ad.String() + } + for _, n := range append(hostAddresses(), a.cfg.TLSNames...) { + n = strings.ToLower(n) + if ad, ok := parseAddr(n); ok { + n = ad.String() + } + if n == h { + return true + } + } + if su, err := url.Parse(a.cfg.ServerURL); err == nil && strings.EqualFold(su.Hostname(), h) { + return true + } + return false +} + +// validTLSURL: what a rack reports must be https on its own URL's host. +func validTLSURL(rc RackController) string { + if rc.TLSURL == "" { + return "" + } + u, err := url.Parse(rc.TLSURL) + host, herr := rackHost(rc.URL) + if err != nil || herr != nil || u.Scheme != "https" || !strings.EqualFold(u.Hostname(), host) || u.Path != "" && u.Path != "/" { + return "" + } + return strings.TrimRight(rc.TLSURL, "/") +} + +// agentTLS is the HTTPS base agents behind this base URL use, "" when none. +func (a *App) agentTLS(base string) string { + if a.ca == nil { + return "" + } + if base == a.cfg.ServerURL { // the embedded rack: the region's own listener + if a.cfg.TLSListen == "" { + return "" + } + u, err := url.Parse(base) + if err != nil || u.Hostname() == "" { + return "" + } + _, port, _ := net.SplitHostPort(a.cfg.TLSListen) + if port == "" || port == "443" { + return "https://" + hostPort(u.Hostname(), "") + } + return "https://" + hostPort(u.Hostname(), port) + } + for _, rc := range a.racksSnapshot() { + if strings.TrimRight(rc.URL, "/") == base { + return validTLSURL(rc) + } + } + return "" +} + +func hostPort(h, port string) string { + if strings.Contains(h, ":") { + h = "[" + h + "]" + } + if port == "" { + return h + } + return h + ":" + port +} + +// agentTLSArgs are the command-line words that switch an agent to HTTPS. +func (a *App) agentTLSArgs(base string) []string { + t := a.agentTLS(base) + if t == "" { + return nil + } + fp := certFingerprint(a.ca.Cert) + return []string{"rollcall.tls=" + t, "rollcall.ca=" + hex.EncodeToString(fp[:])} +} + +// overTLS: the agent's request came over HTTPS (to the region, or to a rack +// controller that says so in X-Forwarded-Proto). +func (a *App) overTLS(r *http.Request) bool { + if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName { + return r.Header.Get("X-Forwarded-Proto") == "https" + } + return r.TLS != nil +} diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go index f255edb..7465cc0 100644 --- a/cmd/rollcall/ca.go +++ b/cmd/rollcall/ca.go @@ -532,16 +532,22 @@ type enrollRequest struct { Rack RackController `json:"rack"` } -func (a *App) signRackCSR(csrPEM, name string) ([]byte, *x509.Certificate, error) { +// parseCSR reads a PEM certificate request; only its key is used. +func parseCSR(csrPEM string) (*x509.CertificateRequest, error) { blk, _ := pem.Decode([]byte(csrPEM)) if blk == nil { - return nil, nil, errors.New("csr is not PEM") + return nil, errors.New("csr is not PEM") } csr, err := x509.ParseCertificateRequest(blk.Bytes) if err != nil { - return nil, nil, err + return nil, err } - if err := csr.CheckSignature(); err != nil { + return csr, csr.CheckSignature() +} + +func (a *App) signRackCSR(csrPEM, name string) ([]byte, *x509.Certificate, error) { + csr, err := parseCSR(csrPEM) + if err != nil { return nil, nil, err } // racks serve no TLS: a client identity only, no names, so a rack cannot pose as the region diff --git a/cmd/rollcall/client.go b/cmd/rollcall/client.go index cb51ba0..c8e8d5b 100644 --- a/cmd/rollcall/client.go +++ b/cmd/rollcall/client.go @@ -39,6 +39,16 @@ func clientTLS(caPEM string) *tls.Config { } } tc := &tls.Config{MinVersion: tls.VersionTLS12} + // a rack controller's certificate (its client one, or the server one it + // shows agents) is never the region, whatever names it carries + tc.VerifyConnection = func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) > 0 { + if cn := cs.PeerCertificates[0].Subject.CommonName; strings.HasPrefix(cn, "rack:") || strings.HasPrefix(cn, rackServerCN) { + return fmt.Errorf("the server shows the certificate of a rack controller (%s), not the region's", cn) + } + } + return nil + } if cert := clientCertificate(); cert != nil { tc.Certificates = []tls.Certificate{*cert} } diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index 4e1a3d5..2e098c9 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -716,6 +716,14 @@ func (a *App) handleJobSelf(w http.ResponseWriter, r *http.Request) { return } sp := j.spec() + if a.overTLS(r) { // an agent speaking HTTPS gets the image over HTTPS too + if m, err := a.store.Get(r.Context(), j.MachineID); err == nil { + base := a.urlFor(m) + if t := a.agentTLS(base); t != "" && strings.HasPrefix(sp.ImageURL, base+"/") { + sp.ImageURL = t + strings.TrimPrefix(sp.ImageURL, base) + } + } + } out := obj{"id": j.ID, "machine_id": j.MachineID, "kind": j.Kind, "spec": sp} if j.Kind == "deploy" { out["files"] = a.renderSeed(j, sp, tok) diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index b211943..b96588f 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -40,7 +40,8 @@ type RackIface struct { type RackController struct { Name string `json:"name"` - URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1 + URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1 + TLSURL string `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents) Kind string `json:"kind"` // embedded | remote Version string `json:"version"` Arch string `json:"arch,omitempty"` // GOARCH, for self-update @@ -90,7 +91,7 @@ type rackResult struct { // ---------------------------------------------------------------- store func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { - rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) if err != nil { return nil, err } @@ -99,7 +100,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { for rows.Next() { var r RackController var ifs, dif []byte - if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { + if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { return nil, err } _ = json.Unmarshal(ifs, &r.Interfaces) @@ -118,9 +119,9 @@ func (s *Store) PutRackController(ctx context.Context, r RackController) error { } ifs, _ := json.Marshal(r.Interfaces) dif, _ := json.Marshal(r.DHCPInterfaces) - _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,'')) - ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`, - r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch) + _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,'')) + ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`, + r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL) return err } @@ -409,10 +410,14 @@ func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) } // appendFor is commission_append with the server URL of the machine's rack. func (a *App) appendFor(base string) string { - if base == a.cfg.ServerURL { - return a.cfg.CommissionAppend + s := a.cfg.CommissionAppend + if base != a.cfg.ServerURL { + s = strings.ReplaceAll(s, a.cfg.ServerURL, base) } - return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base) + if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") { + s += " " + strings.Join(t, " ") + } + return s } // rackAddresses are the rack controllers' own addresses, never handed to machines. @@ -822,6 +827,7 @@ func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack st fail(w, 400, errors.New("url is required")) return } + rc.TLSURL = validTLSURL(rc) // its own host only if err := a.store.PutRackController(r.Context(), rc); err != nil { fail(w, 500, err) return diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 0be7767..2c1c041 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -58,6 +58,10 @@ type RackConfig struct { // rack.pem). The files from rack join are root's; this directory is the // service user's. Default /var/lib/rollcall/rack-cert. CertStateDir string `json:"cert_state_dir,omitempty"` + // TLSListen: HTTPS for the agents (node API, images), with a server + // certificate from the region for the URL's host. Default :443; + // "off" disables it. + TLSListen string `json:"tls_listen,omitempty"` } func (c RackConfig) renewedPath() string { @@ -65,16 +69,19 @@ func (c RackConfig) renewedPath() string { } type rackd struct { - cfg RackConfig - region *url.URL - hc *http.Client // long polls - fast *http.Client - big *http.Client // image downloads, no timeout - gen int64 - tlsc *tls.Config - certMu sync.Mutex - upd rackUpdater - cert *tls.Certificate + srvMu sync.Mutex + srvCert *tls.Certificate // server certificate for agents (HTTPS) + tlsURL string + cfg RackConfig + region *url.URL + hc *http.Client // long polls + fast *http.Client + big *http.Client // image downloads, no timeout + gen int64 + tlsc *tls.Config + certMu sync.Mutex + upd rackUpdater + cert *tls.Certificate headMu sync.Mutex heads map[string]headInfo @@ -469,6 +476,12 @@ func runRack(args []string) error { mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version}) }) + mux.HandleFunc("GET /ca.pem", func(w http.ResponseWriter, req *http.Request) { // agents check it by its fingerprint + http.ServeFile(w, req, cfg.CAFile) + }) + if r.cert != nil && cfg.TLSListen != "off" { + go r.serveTLS(logging(mux)) + } srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute} log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir) return srv.ListenAndServe() @@ -518,7 +531,10 @@ func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, erro } func (r *rackd) register() error { - _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH, + r.srvMu.Lock() + tlsURL := r.tlsURL + r.srvMu.Unlock() + _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, TLSURL: tlsURL, Version: version, Arch: runtime.GOARCH, Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil) return err } @@ -751,3 +767,109 @@ func (r *rackd) prefetch(files []string) { } } } + +// ---------------------------------------------------------------- HTTPS for agents + +func (c RackConfig) serverCertPath() string { + return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "server.pem") +} + +// serverCert keeps a server certificate for the URL's host, from the region. +func (r *rackd) serverCert() (*tls.Certificate, error) { + host, err := rackHost(r.cfg.URL) + if err != nil { + return nil, err + } + p := r.cfg.serverCertPath() + if b, err := os.ReadFile(p); err == nil { + if pair, err := tls.X509KeyPair(b, b); err == nil { + leaf, _ := x509.ParseCertificate(pair.Certificate[0]) + if leaf != nil && time.Until(leaf.NotAfter) > renewBefore && leaf.VerifyHostname(host) == nil { + pair.Leaf = leaf + return &pair, nil + } + } + } + key, csr, err := newCSR(rackServerCN + r.cfg.Name) + if err != nil { + return nil, err + } + var resp struct { + Cert string `json:"cert"` + } + if _, err := r.call(r.fast, "POST", "/api/v1/rack/server-cert", enrollRequest{CSR: csr}, &resp); err != nil { + return nil, err + } + pair, err := tls.X509KeyPair([]byte(resp.Cert), key) + if err != nil { + return nil, err + } + pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) + if err := writeAtomic(p, append([]byte(resp.Cert), key...), 0o600); err != nil { + log.Printf("keep the server certificate: %v (it is used, but asked for again at the next start)", err) + } + return &pair, nil +} + +// serveTLS serves the agents' endpoints over HTTPS once the region issued a +// certificate, re-registers with the HTTPS URL and renews it when it ages. +func (r *rackd) serveTLS(h http.Handler) { + host, err := rackHost(r.cfg.URL) + if err != nil { + log.Printf("https for agents: %v", err) + return + } + addr := r.cfg.TLSListen + if addr == "" { + addr = net.JoinHostPort(host, "443") + } + for { + c, err := r.serverCert() + if err == nil { + r.srvMu.Lock() + r.srvCert = c + r.srvMu.Unlock() + break + } + log.Printf("https for agents: no server certificate yet: %v", err) + time.Sleep(30 * time.Second) + } + srv := &http.Server{Addr: addr, Handler: h, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute, + TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + r.srvMu.Lock() + defer r.srvMu.Unlock() + return r.srvCert, nil + }}} + ln, err := net.Listen("tcp", addr) + if err != nil { + log.Printf("https for agents on %s: %v", addr, err) + return + } + _, port, _ := net.SplitHostPort(addr) + u := "https://" + hostPort(host, "") + if port != "443" { + u = "https://" + hostPort(host, port) + } + r.srvMu.Lock() + r.tlsURL = u + r.srvMu.Unlock() + if err := r.register(); err != nil { + log.Printf("register with %s: %v", u, err) + } + log.Printf("serving agents over HTTPS on %s (%s)", addr, u) + go func() { + for range time.Tick(12 * time.Hour) { + r.srvMu.Lock() + leaf := r.srvCert.Leaf + r.srvMu.Unlock() + if leaf != nil && time.Until(leaf.NotAfter) < renewBefore { + if c, err := r.serverCert(); err == nil { + r.srvMu.Lock() + r.srvCert = c + r.srvMu.Unlock() + } + } + } + }() + log.Printf("https for agents: %v", srv.ServeTLS(ln, "", "")) +} diff --git a/cmd/rollcall/security_test.go b/cmd/rollcall/security_test.go index 40e09e4..971d5d6 100644 --- a/cmd/rollcall/security_test.go +++ b/cmd/rollcall/security_test.go @@ -174,3 +174,25 @@ func TestBMCCertificatePin(t *testing.T) { t.Errorf("another certificate: %v (credentials sent %d times)", err, seen) } } + +// A rack's HTTPS URL is its own host; the region's addresses get no rack certificate. +func TestAgentTLSURLs(t *testing.T) { + ok := RackController{URL: "http://10.213.78.1", TLSURL: "https://10.213.78.1"} + if validTLSURL(ok) != "https://10.213.78.1" { + t.Error("own host refused") + } + for _, bad := range []string{"https://10.0.0.234", "http://10.213.78.1", "https://10.213.78.1/x", "https://evil.example"} { + if validTLSURL(RackController{URL: "http://10.213.78.1", TLSURL: bad}) != "" { + t.Errorf("%s accepted", bad) + } + } + a := &App{cfg: &Config{ServerURL: "http://10.213.77.1", TLSNames: []string{"region.example"}}} + for _, h := range []string{"10.213.77.1", "region.example", "127.0.0.1", "localhost"} { + if !a.isRegionName(h) { + t.Errorf("%s is the region's", h) + } + } + if a.isRegionName("10.213.78.1") { + t.Error("a rack's address taken for the region's") + } +} diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 1111031..e96a42e 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -267,6 +267,7 @@ func (a *App) routes() http.Handler { mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode)) mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew)) + mux.HandleFunc("POST /api/v1/rack/server-cert", a.rackAuth(a.handleRackServerCert)) mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister)) mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState)) mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork)) diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index e4faf66..128a0b0 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -654,6 +654,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string { "hostname=" + deref(m.NodeName), "live-config.nocomponents=user-setup,sudo", "rollcall.server=" + base, "rollcall.node=" + token} + parts = append(parts, a.agentTLSArgs(base)...) if p.Append != "" { parts = append(parts, p.Append) } diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index 773608e..cfa59e9 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -165,6 +165,7 @@ CREATE TABLE IF NOT EXISTS rack_controllers ( registered timestamptz NOT NULL DEFAULT now() ); ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS tls_url text; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz; ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;