c002 built and then could not be reached: "built, but not published: open /etc/hpcod/clusters/c002.crt: read-only file system". The machines were running, the tenant had nothing to open, and the cluster's own name answered with the control plane's front page. The service runs with ProtectSystem=strict and ReadWritePaths on /var/lib/hpcod alone, which is correct - and the registry was in /etc with it. Both halves are written at runtime: a certificate lands there when a cluster is built, and the registry beside it is replaced by rename, so its directory has to be writable too. Widening ReadWritePaths to /etc/hpcod would have handed the service write access to the user file, the database password and the proxy's private key, to fix a path that was simply in the wrong tree. Default is now /var/lib/hpcod/clusters, and B40-publish-cluster.py follows. Existing certificates and the registry were moved on the host, with the absolute paths inside the registry rewritten. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
131 lines
4.6 KiB
Python
Executable File
131 lines
4.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Publish a cluster's portal at c<NNN>.hpc.example.org, or withdraw it.
|
|
|
|
The control plane routes cluster portals by subdomain, resolving the label
|
|
through a registry file (see hpcod/internal/httpapi/tenantproxy.go). This
|
|
script is what writes that file, because the head node cannot: it has no
|
|
access to the control plane's configuration, and should not have.
|
|
|
|
Clusters, not tenants: one tenant may run several at once, so the id here
|
|
is the cluster's.
|
|
|
|
Publishing copies the cluster's own certificate to the control plane, which
|
|
then pins it as the sole trust anchor for that one cluster. Re-running after
|
|
the cluster regenerates its certificate refreshes the copy - necessary,
|
|
because the proxy will refuse a certificate it does not hold.
|
|
|
|
B40-publish-cluster.py 2 10.0.0.62
|
|
B40-publish-cluster.py --remove 2
|
|
"""
|
|
import argparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
import yaml
|
|
|
|
# The portal writes both of these, and it runs with ProtectSystem=strict.
|
|
# They were under /etc until a build reported "built, but not published:
|
|
# read-only file system" - correct, and the wrong place to fix it, since
|
|
# /etc/hpcod also holds the user file, the database password and the
|
|
# proxy's private key.
|
|
REGISTRY = "/var/lib/hpcod/clusters.yaml"
|
|
CERT_DIR = "/var/lib/hpcod/clusters"
|
|
SSH_KEY = "/var/lib/hpcod/.ssh/id_ed25519"
|
|
SSH_USER = "ilker"
|
|
CLUSTER_CERT = "/etc/pki/tls/certs/ood-portal.crt"
|
|
|
|
|
|
def fetch_cert(head: str) -> bytes:
|
|
"""Read the certificate the cluster's portal presents, over SSH."""
|
|
out = subprocess.run(
|
|
[
|
|
"ssh", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no",
|
|
"-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=10",
|
|
"-i", SSH_KEY, f"{SSH_USER}@{head}", f"sudo cat {CLUSTER_CERT}",
|
|
],
|
|
capture_output=True, check=True,
|
|
)
|
|
if b"BEGIN CERTIFICATE" not in out.stdout:
|
|
raise SystemExit(f"{head} did not return a certificate")
|
|
return out.stdout
|
|
|
|
|
|
def load() -> dict:
|
|
try:
|
|
with open(REGISTRY) as fh:
|
|
doc = yaml.safe_load(fh) or {}
|
|
except FileNotFoundError:
|
|
doc = {}
|
|
doc.setdefault("clusters", [])
|
|
return doc
|
|
|
|
|
|
def save(doc: dict) -> None:
|
|
# Written whole and moved into place: the control plane re-reads this
|
|
# file whenever its mtime moves, and a half-written one would take every
|
|
# published cluster offline until the next write.
|
|
d = os.path.dirname(REGISTRY)
|
|
with tempfile.NamedTemporaryFile("w", dir=d, delete=False) as tmp:
|
|
tmp.write("# Written by B40-publish-cluster.py - do not edit by hand.\n")
|
|
yaml.safe_dump(doc, tmp, default_flow_style=False, sort_keys=False)
|
|
name = tmp.name
|
|
os.chmod(name, 0o644)
|
|
os.replace(name, REGISTRY)
|
|
|
|
|
|
def publish(cluster_id: int, head: str) -> None:
|
|
os.makedirs(CERT_DIR, mode=0o755, exist_ok=True)
|
|
cert_path = os.path.join(CERT_DIR, f"c{cluster_id:03d}.crt")
|
|
with open(cert_path, "wb") as fh:
|
|
fh.write(fetch_cert(head))
|
|
os.chmod(cert_path, 0o644)
|
|
|
|
doc = load()
|
|
doc["clusters"] = [t for t in doc["clusters"] if t.get("id") != cluster_id]
|
|
doc["clusters"].append(
|
|
{"id": cluster_id, "portal": f"{head}:443", "cert": cert_path}
|
|
)
|
|
doc["clusters"].sort(key=lambda t: t.get("id", 0))
|
|
save(doc)
|
|
print(f"published c{cluster_id:03d} -> {head}:443")
|
|
print(f" https://c{cluster_id:03d}.hpc.example.org")
|
|
|
|
|
|
def remove(cluster_id: int) -> None:
|
|
doc = load()
|
|
before = len(doc["clusters"])
|
|
doc["clusters"] = [t for t in doc["clusters"] if t.get("id") != cluster_id]
|
|
save(doc)
|
|
# The certificate goes too. Leaving it would let a later cluster reusing
|
|
# this id be trusted against its predecessor's key.
|
|
cert_path = os.path.join(CERT_DIR, f"c{cluster_id:03d}.crt")
|
|
if os.path.exists(cert_path):
|
|
os.remove(cert_path)
|
|
print(f"withdrew c{cluster_id:03d}" if before != len(doc["clusters"])
|
|
else f"c{cluster_id:03d} was not published")
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument("cluster_id", type=int)
|
|
ap.add_argument("head", nargs="?", help="management address of the head node")
|
|
ap.add_argument("--remove", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
if not 1 <= args.cluster_id <= 254:
|
|
raise SystemExit("cluster id must be 1-254")
|
|
|
|
if args.remove:
|
|
remove(args.cluster_id)
|
|
elif not args.head:
|
|
raise SystemExit("publishing needs the head node's address")
|
|
else:
|
|
publish(args.cluster_id, args.head)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|