Agents over HTTPS: rack server certificates, fingerprinted CA
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Failing after 20s

Rack controllers ask the region for a server certificate for their own
URL's host (POST /api/v1/rack/server-cert; CN rack-server:NAME,
ServerAuth only, never one of the region's names or addresses) and serve
the agents' endpoints and images over HTTPS too (tls_listen, default
<host>:443), reporting the HTTPS URL when they register (accepted only
on the rack's own host). The pxelinux command lines name it with the CA
fingerprint (rollcall.tls=, rollcall.ca=); the embedded rack uses the
region's listener. Agents fetch /ca.pem, accept it only by that
fingerprint and then use HTTPS with it, including the image download; a
job fetched over HTTPS gets an HTTPS image URL. No fallback to HTTP once
TLS was asked for; old agents ignore the new words. The CLI refuses a
rack controller's certificate as the region.

PXE itself (TFTP, kernel, initrd) stays unauthenticated: this protects
what agents send and receive from listeners and from an HTTP server
posing as the rack, not from an attacker rewriting boot files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 22:51:17 +01:00
1 parent 74922f7491
commit a9ca326dd2
12 files changed
+443 -29

No files matched your search

+36 -3
View File
@@ -13,11 +13,13 @@ Kernel cmdline options:
rollcall.mode=deploy rollcall.job=.. run rollcall-deploy.py instead (OS deployment / release job)
"""
import glob
import hashlib
import json
import os
import re
import secrets
import socket
import ssl
import string
import subprocess
import sys
@@ -399,6 +401,36 @@ def bmc_inband(user_name):
AGENT_TOKEN = ""
def tls_setup(opts, server):
"""rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
accepted only by that fingerprint; from then on everything goes over HTTPS
with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
return server, None, None
pem = None
for i in range(10):
try:
with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
pem = r.read().decode()
break
except (urllib.error.URLError, OSError) as e:
log(f"ca.pem: {e}, retry {i + 1}/10")
time.sleep(3)
if pem is None:
raise RuntimeError("cannot fetch the CA certificate")
if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
path = "/run/rollcall-ca.pem"
with open(path, "w") as f:
f.write(pem)
log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
return url.rstrip("/"), ssl.create_default_context(cafile=path), path
TLS = None # ssl context once tls_setup ran
def agent_headers():
h = {"Content-Type": "application/json"}
if AGENT_TOKEN:
@@ -409,7 +441,7 @@ def agent_headers():
def bmc_hint(server, macs):
try:
req = urllib.request.Request(f"{server}/api/v1/bmc-hint?macs={','.join(macs)}", headers=agent_headers())
with urllib.request.urlopen(req, timeout=10) as r:
with urllib.request.urlopen(req, timeout=10, context=TLS) as r:
h = json.load(r)
except (urllib.error.URLError, json.JSONDecodeError, OSError):
return None
@@ -439,7 +471,7 @@ def post(server, payload, tries=30):
req = urllib.request.Request(f"{server}/api/v1/commission", data=data, headers=agent_headers(), method="POST")
for i in range(tries):
try:
with urllib.request.urlopen(req, timeout=30) as r:
with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
return json.load(r)
except (urllib.error.URLError, OSError) as e:
log(f"post failed ({e}), retry {i + 1}/{tries}")
@@ -457,7 +489,8 @@ def main():
log("rollcall.server= missing on kernel cmdline; nothing to do")
return 2
server = server.rstrip("/")
global AGENT_TOKEN
global AGENT_TOKEN, TLS
server, TLS, _ = tls_setup(opts, server)
AGENT_TOKEN = opts.get("token") if isinstance(opts.get("token"), str) else ""
user_name = opts.get("bmcuser") or "commissioner"
started = time.time()
+35 -2
View File
@@ -16,6 +16,7 @@ import hashlib
import json
import os
import re
import ssl
import subprocess
import sys
import threading
@@ -62,6 +63,35 @@ def cmdline():
# --------------------------------------------------------------------------- server
TLS, CAFILE = None, None
def tls_setup(opts, server):
"""rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
accepted only by that fingerprint; from then on everything goes over HTTPS
with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
return server, None, None
pem = None
for i in range(10):
try:
with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
pem = r.read().decode()
break
except (urllib.error.URLError, OSError) as e:
log(f"ca.pem: {e}, retry {i + 1}/10")
time.sleep(3)
if pem is None:
raise RuntimeError("cannot fetch the CA certificate")
if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
path = "/run/rollcall-ca.pem"
with open(path, "w") as f:
f.write(pem)
log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
return url.rstrip("/"), ssl.create_default_context(cafile=path), path
def api(method, path, body=None, tries=20):
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(SERVER + path, data=data, method=method,
@@ -69,7 +99,7 @@ def api(method, path, body=None, tries=20):
last = None
for i in range(tries):
try:
with urllib.request.urlopen(req, timeout=30) as r:
with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
return json.load(r)
except urllib.error.HTTPError as e:
if 400 <= e.code < 500:
@@ -196,7 +226,8 @@ def write_image(spec, disk, prog):
["bash", "-c", 'set -o pipefail; pv -n -f -i 2 -s "$1" 2>"$2" | zstd -dc -T0 '
'| dd of="$3" bs=4M iflag=fullblock oflag=direct conv=fsync status=none', "write", str(size), pvf, disk],
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused", "--", url],
cacert = ["--cacert", CAFILE] if CAFILE and url.startswith("https://") else []
curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused"] + cacert + ["--", url],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
log(f"writing {url} -> {disk}")
h, pct, shown = hashlib.sha256(), 0, 0.0
@@ -370,6 +401,8 @@ def main():
global SERVER, TOKEN
opts = cmdline()
SERVER = (opts.get("server") or "").rstrip("/")
global TLS, CAFILE
SERVER, TLS, CAFILE = tls_setup(opts, SERVER)
TOKEN = opts.get("job") if isinstance(opts.get("job"), str) else ""
if not SERVER or not TOKEN:
log("rollcall.server= and rollcall.job= required")