Agents over HTTPS: rack server certificates, fingerprinted CA

Rack controllers ask the region for a server certificate for their own
URL's host (POST /api/v1/rack/server-cert; CN rack-server:NAME,
ServerAuth only, never one of the region's names or addresses) and serve
the agents' endpoints and images over HTTPS too (tls_listen, default
<host>:443), reporting the HTTPS URL when they register (accepted only
on the rack's own host). The pxelinux command lines name it with the CA
fingerprint (rollcall.tls=, rollcall.ca=); the embedded rack uses the
region's listener. Agents fetch /ca.pem, accept it only by that
fingerprint and then use HTTPS with it, including the image download; a
job fetched over HTTPS gets an HTTPS image URL. No fallback to HTTP once
TLS was asked for; old agents ignore the new words. The CLI refuses a
rack controller's certificate as the region.

PXE itself (TFTP, kernel, initrd) stays unauthenticated: this protects
what agents send and receive from listeners and from an HTTP server
posing as the rack, not from an attacker rewriting boot files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

agent/rollcall-agent.py

@@ -13,11 +13,13 @@ Kernel cmdline options:
   rollcall.mode=deploy rollcall.job=.. run rollcall-deploy.py instead (OS deployment / release job)
 """
 import glob
+import hashlib
 import json
 import os
 import re
 import secrets
 import socket
+import ssl
 import string
 import subprocess
 import sys
@@ -399,6 +401,36 @@ def bmc_inband(user_name):
 AGENT_TOKEN = ""
 
 
+def tls_setup(opts, server):
+    """rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
+    accepted only by that fingerprint; from then on everything goes over HTTPS
+    with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
+    url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
+    if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
+        return server, None, None
+    pem = None
+    for i in range(10):
+        try:
+            with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
+                pem = r.read().decode()
+            break
+        except (urllib.error.URLError, OSError) as e:
+            log(f"ca.pem: {e}, retry {i + 1}/10")
+            time.sleep(3)
+    if pem is None:
+        raise RuntimeError("cannot fetch the CA certificate")
+    if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
+        raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
+    path = "/run/rollcall-ca.pem"
+    with open(path, "w") as f:
+        f.write(pem)
+    log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
+    return url.rstrip("/"), ssl.create_default_context(cafile=path), path
+
+
+TLS = None  # ssl context once tls_setup ran
+
+
 def agent_headers():
     h = {"Content-Type": "application/json"}
     if AGENT_TOKEN:
@@ -409,7 +441,7 @@ def agent_headers():
 def bmc_hint(server, macs):
     try:
         req = urllib.request.Request(f"{server}/api/v1/bmc-hint?macs={','.join(macs)}", headers=agent_headers())
-        with urllib.request.urlopen(req, timeout=10) as r:
+        with urllib.request.urlopen(req, timeout=10, context=TLS) as r:
             h = json.load(r)
     except (urllib.error.URLError, json.JSONDecodeError, OSError):
         return None
@@ -439,7 +471,7 @@ def post(server, payload, tries=30):
     req = urllib.request.Request(f"{server}/api/v1/commission", data=data, headers=agent_headers(), method="POST")
     for i in range(tries):
         try:
-            with urllib.request.urlopen(req, timeout=30) as r:
+            with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
                 return json.load(r)
         except (urllib.error.URLError, OSError) as e:
             log(f"post failed ({e}), retry {i + 1}/{tries}")
@@ -457,7 +489,8 @@ def main():
         log("rollcall.server= missing on kernel cmdline; nothing to do")
         return 2
     server = server.rstrip("/")
-    global AGENT_TOKEN
+    global AGENT_TOKEN, TLS
+    server, TLS, _ = tls_setup(opts, server)
     AGENT_TOKEN = opts.get("token") if isinstance(opts.get("token"), str) else ""
     user_name = opts.get("bmcuser") or "commissioner"
     started = time.time()

agent/rollcall-deploy.py

@@ -16,6 +16,7 @@ import hashlib
 import json
 import os
 import re
+import ssl
 import subprocess
 import sys
 import threading
@@ -62,6 +63,35 @@ def cmdline():
 
 
 # --------------------------------------------------------------------------- server
+TLS, CAFILE = None, None
+
+def tls_setup(opts, server):
+    """rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
+    accepted only by that fingerprint; from then on everything goes over HTTPS
+    with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
+    url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
+    if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
+        return server, None, None
+    pem = None
+    for i in range(10):
+        try:
+            with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
+                pem = r.read().decode()
+            break
+        except (urllib.error.URLError, OSError) as e:
+            log(f"ca.pem: {e}, retry {i + 1}/10")
+            time.sleep(3)
+    if pem is None:
+        raise RuntimeError("cannot fetch the CA certificate")
+    if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
+        raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
+    path = "/run/rollcall-ca.pem"
+    with open(path, "w") as f:
+        f.write(pem)
+    log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
+    return url.rstrip("/"), ssl.create_default_context(cafile=path), path
+
+
 def api(method, path, body=None, tries=20):
     data = json.dumps(body).encode() if body is not None else None
     req = urllib.request.Request(SERVER + path, data=data, method=method,
@@ -69,7 +99,7 @@ def api(method, path, body=None, tries=20):
     last = None
     for i in range(tries):
         try:
-            with urllib.request.urlopen(req, timeout=30) as r:
+            with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
                 return json.load(r)
         except urllib.error.HTTPError as e:
             if 400 <= e.code < 500:
@@ -196,7 +226,8 @@ def write_image(spec, disk, prog):
         ["bash", "-c", 'set -o pipefail; pv -n -f -i 2 -s "$1" 2>"$2" | zstd -dc -T0 '
          '| dd of="$3" bs=4M iflag=fullblock oflag=direct conv=fsync status=none', "write", str(size), pvf, disk],
         stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-    curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused", "--", url],
+    cacert = ["--cacert", CAFILE] if CAFILE and url.startswith("https://") else []
+    curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused"] + cacert + ["--", url],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     log(f"writing {url} -> {disk}")
     h, pct, shown = hashlib.sha256(), 0, 0.0
@@ -370,6 +401,8 @@ def main():
     global SERVER, TOKEN
     opts = cmdline()
     SERVER = (opts.get("server") or "").rstrip("/")
+    global TLS, CAFILE
+    SERVER, TLS, CAFILE = tls_setup(opts, SERVER)
     TOKEN = opts.get("job") if isinstance(opts.get("job"), str) else ""
     if not SERVER or not TOKEN:
         log("rollcall.server= and rollcall.job= required")

cmd/rollcall/agenttls.go

@@ -0,0 +1,171 @@
+package main
+
+// TLS for the agents (commissioning, deploy, stateless nodes). Rack
+// controllers serve the node API and images over HTTPS too, with a server
+// certificate the region issues for the rack's own address only (never one
+// of the region's). The pxelinux command line names the HTTPS URL and the
+// CA's fingerprint (rollcall.tls=, rollcall.ca=); an agent fetches the CA
+// certificate, accepts it only by that fingerprint, and then talks HTTPS
+// with it. The embedded rack uses the region's own certificate.
+//
+// This keeps what agents send and receive (BMC passwords, seed files) from
+// anyone who can only listen, and from an HTTP server posing as the rack.
+// PXE itself (TFTP, the kernel, the initrd) stays unauthenticated: an
+// attacker who can rewrite the boot files on the wire is not stopped here;
+// that takes UEFI Secure Boot with signed images.
+
+import (
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// rackServerCN is the subject of a rack controller's server certificate;
+// regionLeaf and the CLI refuse it as the region.
+const rackServerCN = "rack-server:"
+
+// handleRackServerCert issues a rack's server certificate for the host of its
+// registered URL: {"csr"} -> {"cert"}.
+func (a *App) handleRackServerCert(w http.ResponseWriter, r *http.Request, rack string) {
+	if a.peerRack(r) != rack || a.ca == nil {
+		fail(w, 401, errors.New("a server certificate needs the rack's own certificate"))
+		return
+	}
+	rc, ok := a.racksSnapshot()[rack]
+	if !ok {
+		fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
+		return
+	}
+	host, err := rackHost(rc.URL)
+	if err != nil {
+		fail(w, 400, err)
+		return
+	}
+	if a.isRegionName(host) {
+		fail(w, 403, fmt.Errorf("%s is the region's own address: no rack certificate for it", host))
+		return
+	}
+	var req enrollRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
+		fail(w, 400, err)
+		return
+	}
+	csr, err := parseCSR(req.CSR)
+	if err != nil {
+		fail(w, 400, err)
+		return
+	}
+	certPEM, cert, err := a.ca.sign(csr.PublicKey, rackServerCN+rack, []string{host}, x509.ExtKeyUsageServerAuth)
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	log.Printf("rack controller %s: server certificate for %s until %s", rack, host, cert.NotAfter.Format("2006-01-02"))
+	writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.PEM)})
+}
+
+// rackHost is the host part of a rack URL (an address or a name).
+func rackHost(raw string) (string, error) {
+	u, err := url.Parse(raw)
+	if err != nil || u.Hostname() == "" {
+		return "", fmt.Errorf("rack URL %q has no host", raw)
+	}
+	return u.Hostname(), nil
+}
+
+// isRegionName: one of the names or addresses the region's own certificate covers.
+func (a *App) isRegionName(host string) bool {
+	h := strings.ToLower(strings.Trim(host, "[]"))
+	if ad, ok := parseAddr(h); ok {
+		h = ad.String()
+	}
+	for _, n := range append(hostAddresses(), a.cfg.TLSNames...) {
+		n = strings.ToLower(n)
+		if ad, ok := parseAddr(n); ok {
+			n = ad.String()
+		}
+		if n == h {
+			return true
+		}
+	}
+	if su, err := url.Parse(a.cfg.ServerURL); err == nil && strings.EqualFold(su.Hostname(), h) {
+		return true
+	}
+	return false
+}
+
+// validTLSURL: what a rack reports must be https on its own URL's host.
+func validTLSURL(rc RackController) string {
+	if rc.TLSURL == "" {
+		return ""
+	}
+	u, err := url.Parse(rc.TLSURL)
+	host, herr := rackHost(rc.URL)
+	if err != nil || herr != nil || u.Scheme != "https" || !strings.EqualFold(u.Hostname(), host) || u.Path != "" && u.Path != "/" {
+		return ""
+	}
+	return strings.TrimRight(rc.TLSURL, "/")
+}
+
+// agentTLS is the HTTPS base agents behind this base URL use, "" when none.
+func (a *App) agentTLS(base string) string {
+	if a.ca == nil {
+		return ""
+	}
+	if base == a.cfg.ServerURL { // the embedded rack: the region's own listener
+		if a.cfg.TLSListen == "" {
+			return ""
+		}
+		u, err := url.Parse(base)
+		if err != nil || u.Hostname() == "" {
+			return ""
+		}
+		_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
+		if port == "" || port == "443" {
+			return "https://" + hostPort(u.Hostname(), "")
+		}
+		return "https://" + hostPort(u.Hostname(), port)
+	}
+	for _, rc := range a.racksSnapshot() {
+		if strings.TrimRight(rc.URL, "/") == base {
+			return validTLSURL(rc)
+		}
+	}
+	return ""
+}
+
+func hostPort(h, port string) string {
+	if strings.Contains(h, ":") {
+		h = "[" + h + "]"
+	}
+	if port == "" {
+		return h
+	}
+	return h + ":" + port
+}
+
+// agentTLSArgs are the command-line words that switch an agent to HTTPS.
+func (a *App) agentTLSArgs(base string) []string {
+	t := a.agentTLS(base)
+	if t == "" {
+		return nil
+	}
+	fp := certFingerprint(a.ca.Cert)
+	return []string{"rollcall.tls=" + t, "rollcall.ca=" + hex.EncodeToString(fp[:])}
+}
+
+// overTLS: the agent's request came over HTTPS (to the region, or to a rack
+// controller that says so in X-Forwarded-Proto).
+func (a *App) overTLS(r *http.Request) bool {
+	if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName {
+		return r.Header.Get("X-Forwarded-Proto") == "https"
+	}
+	return r.TLS != nil
+}

cmd/rollcall/ca.go

@@ -532,16 +532,22 @@ type enrollRequest struct {
 	Rack   RackController `json:"rack"`
 }
 
-func (a *App) signRackCSR(csrPEM, name string) ([]byte, *x509.Certificate, error) {
+// parseCSR reads a PEM certificate request; only its key is used.
+func parseCSR(csrPEM string) (*x509.CertificateRequest, error) {
 	blk, _ := pem.Decode([]byte(csrPEM))
 	if blk == nil {
-		return nil, nil, errors.New("csr is not PEM")
+		return nil, errors.New("csr is not PEM")
 	}
 	csr, err := x509.ParseCertificateRequest(blk.Bytes)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	if err := csr.CheckSignature(); err != nil {
+	return csr, csr.CheckSignature()
+}
+
+func (a *App) signRackCSR(csrPEM, name string) ([]byte, *x509.Certificate, error) {
+	csr, err := parseCSR(csrPEM)
+	if err != nil {
 		return nil, nil, err
 	}
 	// racks serve no TLS: a client identity only, no names, so a rack cannot pose as the region

cmd/rollcall/client.go

@@ -39,6 +39,16 @@ func clientTLS(caPEM string) *tls.Config {
 		}
 	}
 	tc := &tls.Config{MinVersion: tls.VersionTLS12}
+	// a rack controller's certificate (its client one, or the server one it
+	// shows agents) is never the region, whatever names it carries
+	tc.VerifyConnection = func(cs tls.ConnectionState) error {
+		if len(cs.PeerCertificates) > 0 {
+			if cn := cs.PeerCertificates[0].Subject.CommonName; strings.HasPrefix(cn, "rack:") || strings.HasPrefix(cn, rackServerCN) {
+				return fmt.Errorf("the server shows the certificate of a rack controller (%s), not the region's", cn)
+			}
+		}
+		return nil
+	}
 	if cert := clientCertificate(); cert != nil {
 		tc.Certificates = []tls.Certificate{*cert}
 	}

cmd/rollcall/deploy.go

@@ -716,6 +716,14 @@ func (a *App) handleJobSelf(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	sp := j.spec()
+	if a.overTLS(r) { // an agent speaking HTTPS gets the image over HTTPS too
+		if m, err := a.store.Get(r.Context(), j.MachineID); err == nil {
+			base := a.urlFor(m)
+			if t := a.agentTLS(base); t != "" && strings.HasPrefix(sp.ImageURL, base+"/") {
+				sp.ImageURL = t + strings.TrimPrefix(sp.ImageURL, base)
+			}
+		}
+	}
 	out := obj{"id": j.ID, "machine_id": j.MachineID, "kind": j.Kind, "spec": sp}
 	if j.Kind == "deploy" {
 		out["files"] = a.renderSeed(j, sp, tok)

cmd/rollcall/rackctl.go

@@ -40,7 +40,8 @@ type RackIface struct {
 
 type RackController struct {
 	Name           string      `json:"name"`
-	URL            string      `json:"url"`  // how machines reach it, e.g. http://10.213.78.1
+	URL            string      `json:"url"`               // how machines reach it, e.g. http://10.213.78.1
+	TLSURL         string      `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents)
 	Kind           string      `json:"kind"` // embedded | remote
 	Version        string      `json:"version"`
 	Arch           string      `json:"arch,omitempty"` // GOARCH, for self-update
@@ -90,7 +91,7 @@ type rackResult struct {
 // ---------------------------------------------------------------- store
 
 func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
-	rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
+	rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
 	if err != nil {
 		return nil, err
 	}
@@ -99,7 +100,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
 	for rows.Next() {
 		var r RackController
 		var ifs, dif []byte
-		if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
+		if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
 			return nil, err
 		}
 		_ = json.Unmarshal(ifs, &r.Interfaces)
@@ -118,9 +119,9 @@ func (s *Store) PutRackController(ctx context.Context, r RackController) error {
 	}
 	ifs, _ := json.Marshal(r.Interfaces)
 	dif, _ := json.Marshal(r.DHCPInterfaces)
-	_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
-		ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
-		r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
+	_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,''))
+		ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`,
+		r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL)
 	return err
 }
 
@@ -409,10 +410,14 @@ func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
 
 // appendFor is commission_append with the server URL of the machine's rack.
 func (a *App) appendFor(base string) string {
-	if base == a.cfg.ServerURL {
-		return a.cfg.CommissionAppend
+	s := a.cfg.CommissionAppend
+	if base != a.cfg.ServerURL {
+		s = strings.ReplaceAll(s, a.cfg.ServerURL, base)
 	}
-	return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
+	if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") {
+		s += " " + strings.Join(t, " ")
+	}
+	return s
 }
 
 // rackAddresses are the rack controllers' own addresses, never handed to machines.
@@ -822,6 +827,7 @@ func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack st
 		fail(w, 400, errors.New("url is required"))
 		return
 	}
+	rc.TLSURL = validTLSURL(rc) // its own host only
 	if err := a.store.PutRackController(r.Context(), rc); err != nil {
 		fail(w, 500, err)
 		return

cmd/rollcall/rackd.go

@@ -58,6 +58,10 @@ type RackConfig struct {
 	// rack.pem). The files from rack join are root's; this directory is the
 	// service user's. Default /var/lib/rollcall/rack-cert.
 	CertStateDir string `json:"cert_state_dir,omitempty"`
+	// TLSListen: HTTPS for the agents (node API, images), with a server
+	// certificate from the region for the URL's host. Default <url host>:443;
+	// "off" disables it.
+	TLSListen string `json:"tls_listen,omitempty"`
 }
 
 func (c RackConfig) renewedPath() string {
@@ -65,16 +69,19 @@ func (c RackConfig) renewedPath() string {
 }
 
 type rackd struct {
-	cfg    RackConfig
-	region *url.URL
-	hc     *http.Client // long polls
-	fast   *http.Client
-	big    *http.Client // image downloads, no timeout
-	gen    int64
-	tlsc   *tls.Config
-	certMu sync.Mutex
-	upd    rackUpdater
-	cert   *tls.Certificate
+	srvMu   sync.Mutex
+	srvCert *tls.Certificate // server certificate for agents (HTTPS)
+	tlsURL  string
+	cfg     RackConfig
+	region  *url.URL
+	hc      *http.Client // long polls
+	fast    *http.Client
+	big     *http.Client // image downloads, no timeout
+	gen     int64
+	tlsc    *tls.Config
+	certMu  sync.Mutex
+	upd     rackUpdater
+	cert    *tls.Certificate
 
 	headMu sync.Mutex
 	heads  map[string]headInfo
@@ -469,6 +476,12 @@ func runRack(args []string) error {
 	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
 		writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version})
 	})
+	mux.HandleFunc("GET /ca.pem", func(w http.ResponseWriter, req *http.Request) { // agents check it by its fingerprint
+		http.ServeFile(w, req, cfg.CAFile)
+	})
+	if r.cert != nil && cfg.TLSListen != "off" {
+		go r.serveTLS(logging(mux))
+	}
 	srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
 	log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir)
 	return srv.ListenAndServe()
@@ -518,7 +531,10 @@ func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, erro
 }
 
 func (r *rackd) register() error {
-	_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH,
+	r.srvMu.Lock()
+	tlsURL := r.tlsURL
+	r.srvMu.Unlock()
+	_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, TLSURL: tlsURL, Version: version, Arch: runtime.GOARCH,
 		Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil)
 	return err
 }
@@ -751,3 +767,109 @@ func (r *rackd) prefetch(files []string) {
 		}
 	}
 }
+
+// ---------------------------------------------------------------- HTTPS for agents
+
+func (c RackConfig) serverCertPath() string {
+	return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "server.pem")
+}
+
+// serverCert keeps a server certificate for the URL's host, from the region.
+func (r *rackd) serverCert() (*tls.Certificate, error) {
+	host, err := rackHost(r.cfg.URL)
+	if err != nil {
+		return nil, err
+	}
+	p := r.cfg.serverCertPath()
+	if b, err := os.ReadFile(p); err == nil {
+		if pair, err := tls.X509KeyPair(b, b); err == nil {
+			leaf, _ := x509.ParseCertificate(pair.Certificate[0])
+			if leaf != nil && time.Until(leaf.NotAfter) > renewBefore && leaf.VerifyHostname(host) == nil {
+				pair.Leaf = leaf
+				return &pair, nil
+			}
+		}
+	}
+	key, csr, err := newCSR(rackServerCN + r.cfg.Name)
+	if err != nil {
+		return nil, err
+	}
+	var resp struct {
+		Cert string `json:"cert"`
+	}
+	if _, err := r.call(r.fast, "POST", "/api/v1/rack/server-cert", enrollRequest{CSR: csr}, &resp); err != nil {
+		return nil, err
+	}
+	pair, err := tls.X509KeyPair([]byte(resp.Cert), key)
+	if err != nil {
+		return nil, err
+	}
+	pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
+	if err := writeAtomic(p, append([]byte(resp.Cert), key...), 0o600); err != nil {
+		log.Printf("keep the server certificate: %v (it is used, but asked for again at the next start)", err)
+	}
+	return &pair, nil
+}
+
+// serveTLS serves the agents' endpoints over HTTPS once the region issued a
+// certificate, re-registers with the HTTPS URL and renews it when it ages.
+func (r *rackd) serveTLS(h http.Handler) {
+	host, err := rackHost(r.cfg.URL)
+	if err != nil {
+		log.Printf("https for agents: %v", err)
+		return
+	}
+	addr := r.cfg.TLSListen
+	if addr == "" {
+		addr = net.JoinHostPort(host, "443")
+	}
+	for {
+		c, err := r.serverCert()
+		if err == nil {
+			r.srvMu.Lock()
+			r.srvCert = c
+			r.srvMu.Unlock()
+			break
+		}
+		log.Printf("https for agents: no server certificate yet: %v", err)
+		time.Sleep(30 * time.Second)
+	}
+	srv := &http.Server{Addr: addr, Handler: h, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute,
+		TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			r.srvMu.Lock()
+			defer r.srvMu.Unlock()
+			return r.srvCert, nil
+		}}}
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Printf("https for agents on %s: %v", addr, err)
+		return
+	}
+	_, port, _ := net.SplitHostPort(addr)
+	u := "https://" + hostPort(host, "")
+	if port != "443" {
+		u = "https://" + hostPort(host, port)
+	}
+	r.srvMu.Lock()
+	r.tlsURL = u
+	r.srvMu.Unlock()
+	if err := r.register(); err != nil {
+		log.Printf("register with %s: %v", u, err)
+	}
+	log.Printf("serving agents over HTTPS on %s (%s)", addr, u)
+	go func() {
+		for range time.Tick(12 * time.Hour) {
+			r.srvMu.Lock()
+			leaf := r.srvCert.Leaf
+			r.srvMu.Unlock()
+			if leaf != nil && time.Until(leaf.NotAfter) < renewBefore {
+				if c, err := r.serverCert(); err == nil {
+					r.srvMu.Lock()
+					r.srvCert = c
+					r.srvMu.Unlock()
+				}
+			}
+		}
+	}()
+	log.Printf("https for agents: %v", srv.ServeTLS(ln, "", ""))
+}

cmd/rollcall/security_test.go

@@ -174,3 +174,25 @@ func TestBMCCertificatePin(t *testing.T) {
 		t.Errorf("another certificate: %v (credentials sent %d times)", err, seen)
 	}
 }
+
+// A rack's HTTPS URL is its own host; the region's addresses get no rack certificate.
+func TestAgentTLSURLs(t *testing.T) {
+	ok := RackController{URL: "http://10.213.78.1", TLSURL: "https://10.213.78.1"}
+	if validTLSURL(ok) != "https://10.213.78.1" {
+		t.Error("own host refused")
+	}
+	for _, bad := range []string{"https://10.0.0.234", "http://10.213.78.1", "https://10.213.78.1/x", "https://evil.example"} {
+		if validTLSURL(RackController{URL: "http://10.213.78.1", TLSURL: bad}) != "" {
+			t.Errorf("%s accepted", bad)
+		}
+	}
+	a := &App{cfg: &Config{ServerURL: "http://10.213.77.1", TLSNames: []string{"region.example"}}}
+	for _, h := range []string{"10.213.77.1", "region.example", "127.0.0.1", "localhost"} {
+		if !a.isRegionName(h) {
+			t.Errorf("%s is the region's", h)
+		}
+	}
+	if a.isRegionName("10.213.78.1") {
+		t.Error("a rack's address taken for the region's")
+	}
+}

cmd/rollcall/server.go

@@ -267,6 +267,7 @@ func (a *App) routes() http.Handler {
 	mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode))
 	mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only
 	mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew))
+	mux.HandleFunc("POST /api/v1/rack/server-cert", a.rackAuth(a.handleRackServerCert))
 	mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
 	mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
 	mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))

cmd/rollcall/stateless.go

@@ -654,6 +654,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
 		"hostname=" + deref(m.NodeName),
 		"live-config.nocomponents=user-setup,sudo",
 		"rollcall.server=" + base, "rollcall.node=" + token}
+	parts = append(parts, a.agentTLSArgs(base)...)
 	if p.Append != "" {
 		parts = append(parts, p.Append)
 	}

cmd/rollcall/store.go

@@ -165,6 +165,7 @@ CREATE TABLE IF NOT EXISTS rack_controllers (
   registered      timestamptz NOT NULL DEFAULT now()
 );
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text;
+ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS tls_url text;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz;
 ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;