Deployments trust the CA on RHEL-family systems too

cloud-init's ca_certs ran update-ca-certificates on Rocky and failed, so
those machines never trusted the Rollcall CA. The CA is now written into
both trust-anchor directories and a bootcmd rebuilds the store with
update-ca-trust or update-ca-certificates.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -87,6 +87,9 @@ publish a tag that has no section here.
 - Release notes from `CHANGELOG.md`.
 
 ### Fixed
+- RHEL-family deployments (Rocky, Alma, CentOS) did not trust the Rollcall CA: cloud-init's
+  `ca_certs` ran `update-ca-certificates` there and failed. The CA now goes into the image's trust
+  anchors and a `bootcmd` rebuilds the store with the distribution's tool.
 - A deploy whose power command was lost (rack or region restarting) stayed pending forever and the
   event looked like a success: power errors are now part of the event (`pxe boot failed`), and a
   job the machine has not started within `job_start_timeout` (default 30m, room for servers that

README.md

@@ -445,9 +445,11 @@ plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and
   first connect and keeps the CA in `~/.config/rollcall/client.json` once you
   confirm it (`rollcall ca show` prints it on the region). `/ca.pem` serves
   the certificate; the web login links to it.
-* **Machines** trust it: deployments add it with cloud-init `ca_certs`
-  (allowed on RHEL-family images, where cloud-init marks the module
-  unverified), stateless overlays get it as `.CA`. `rollcall ca issue NAME
+* **Machines** trust it: deployments write the CA into the trust anchors of
+  both families (`/usr/local/share/ca-certificates`, `/etc/pki/ca-trust/source/anchors`)
+  and rebuild the store at every boot (`update-ca-certificates` or
+  `update-ca-trust`; cloud-init's `ca_certs` fails on RHEL-family systems);
+  stateless overlays get it as `.CA`. `rollcall ca issue NAME
   -host h1,h2 [-client]` issues leaves for anything else (Slurm REST, a
   registry, a web front end).
 * **Delegation.** The root may sign one level of intermediate CAs, each

cmd/rollcall/ca.go

@@ -9,7 +9,7 @@ package main
 //     expiry, address in one word) and get a client+server certificate whose
 //     common name is their name; from then on they talk to the region over
 //     mutual TLS and the shared rack_secret is not needed;
-//   - deployed machines trust it (cloud-init ca_certs), stateless overlays get
+//   - deployed machines trust it (anchors in the image + bootcmd), stateless overlays get
 //     it as .CA, and `rollcall ca issue` hands out leaves for anything else.
 //
 // Nodes still boot over plain HTTP: PXE firmware and the live image have no

cmd/rollcall/deploy.go

@@ -291,12 +291,12 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
 	}
 	var frags []TagDef
 	if a.ca != nil {
-		var cc strings.Builder
-		cc.WriteString("#cloud-config\n# rollcall: trust the installation's certificate authority\nca_certs:\n  trusted:\n    - |\n")
-		for _, l := range strings.Split(strings.TrimSpace(string(a.ca.PEM)), "\n") {
-			cc.WriteString("      " + l + "\n")
-		}
-		frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: cc.String()})
+		// The CA file itself is written into the image (seed files below); this rebuilds
+		// the trust store on every boot, early, with whatever the distribution has.
+		// cloud-init's ca_certs runs update-ca-certificates even on RHEL-family systems
+		// and fails there, so it is not used.
+		frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: "#cloud-config\n# rollcall: trust the installation's certificate authority\nbootcmd:\n" +
+			"  - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n"})
 	}
 	if merr == nil {
 		if tf, err := a.tagFragments(context.Background(), m); err == nil {
@@ -309,8 +309,12 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
 	files := []seedFile{
 		{Path: "var/lib/cloud/seed/nocloud/meta-data", Mode: "0600", Content: meta},
 		{Path: "var/lib/cloud/seed/nocloud/user-data", Mode: "0600", Content: ud},
-		// ca_certs is "not verified" on some distros (Rocky, Alma) and skipped unless allowed
-		{Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\nunverified_modules: [ ca_certs ]\n"},
+		{Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"},
+	}
+	if a.ca != nil { // both layouts; the bootcmd above picks the one the distribution uses
+		files = append(files,
+			seedFile{Path: "usr/local/share/ca-certificates/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)},
+			seedFile{Path: "etc/pki/ca-trust/source/anchors/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)})
 	}
 	if sp.NetworkConfig != "" {
 		files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig})