Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

772 lines
26 KiB
Go

package setup
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/pem"
"fmt"
"io"
"math/big"
"net"
"os"
"os/exec"
"os/user"
"regexp"
"strings"
"time"
"golang.org/x/crypto/bcrypt"
"golang.org/x/crypto/ssh"
"github.com/ilkermanap/hpcod/internal/pki"
)
const owner = ServiceID + ":" + ServiceID
// A mirror that drops one request out of a few hundred fails a whole
// install without this; apt's default is not to retry at all.
const aptRetries = "Acquire::Retries=3"
const rootGroup = "root:" + ServiceID
// stepSite records the site where every later step - and every later run
// of this binary - reads it from.
func stepSite(_ context.Context, e *Env) error {
e.Site.ApplySim()
if errs := e.Site.Validate(); len(errs) > 0 {
var b strings.Builder
for _, err := range errs {
b.WriteString("\n " + err.Error())
}
return fmt.Errorf("the site is not complete:%s", b.String())
}
// Owned by root alone for now: on a first install the hpcod group does
// not exist until the account step, which gives the directory to it.
if err := e.mkdir(EtcDir, 0o750, ""); err != nil {
return err
}
if err := e.writeFile(SiteFile, []byte(e.Site.Render()), 0o644, "root:root"); err != nil {
return err
}
e.Logf("site written to %s", SiteFile)
return nil
}
// stepPackages installs what the control plane drives: tofu, ansible,
// PostgreSQL. Everything else is in this binary.
func stepPackages(ctx context.Context, e *Env) error {
osr, err := os.ReadFile("/etc/os-release")
if err != nil {
return err
}
id := osRelease(string(osr), "ID") + " " + osRelease(string(osr), "ID_LIKE")
if !strings.Contains(" "+id+" ", " debian ") && !strings.Contains(" "+id+" ", " ubuntu ") {
return fmt.Errorf("this installs with apt, and %s is not Debian or Ubuntu.\n"+
"Install postgresql, ansible and tofu by hand, then run setup with --skip packages",
osRelease(string(osr), "PRETTY_NAME"))
}
env := []string{"DEBIAN_FRONTEND=noninteractive"}
// ansible, not ansible-core: the roles use ansible.posix, which only the
// full package carries.
want := []string{"postgresql", "ansible", "openssh-client", "python3-yaml", "curl", "gnupg", "ca-certificates"}
if e.Site.Sim() {
want = append(want, "qemu-system-x86", "qemu-utils", "genisoimage", "nftables", "iproute2")
}
var need []string
for _, p := range want {
if out, _ := e.run(ctx, "", "dpkg-query", "-W", "-f=${Status}", p); !strings.Contains(out, "ok installed") {
need = append(need, p)
}
}
if len(need) > 0 {
e.Logf("installing %s", strings.Join(need, " "))
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "update", "-qq"); err != nil {
return err
}
args := append([]string{"-o", aptRetries, "install", "-y", "-qq", "--no-install-recommends"}, need...)
if err := e.stream(ctx, env, "", "apt-get", args...); err != nil {
return err
}
}
// OpenTofu from its own repository; distributions do not package it.
if _, err := exec.LookPath("tofu"); err != nil {
e.Logf("adding the OpenTofu repository")
if err := e.mkdir("/etc/apt/keyrings", 0o755, "root:root"); err != nil {
return err
}
for _, k := range []struct{ url, path string }{
{"https://get.opentofu.org/opentofu.gpg", "/etc/apt/keyrings/opentofu.gpg"},
{"https://packages.opentofu.org/opentofu/tofu/gpgkey", "/etc/apt/keyrings/opentofu-repo.gpg"},
} {
raw, err := e.run(ctx, "", "sh", "-c",
fmt.Sprintf("curl -fsSL %s | gpg --no-tty --batch --yes --dearmor -o %s && chmod a+r %s", k.url, k.path, k.path))
if err != nil {
return fmt.Errorf("fetching %s: %w %s", k.url, err, raw)
}
}
list := "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] " +
"https://packages.opentofu.org/opentofu/tofu/any/ any main\n"
if err := e.writeFile("/etc/apt/sources.list.d/opentofu.list", []byte(list), 0o644, "root:root"); err != nil {
return err
}
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "update", "-qq"); err != nil {
return err
}
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "install", "-y", "-qq", "tofu"); err != nil {
return err
}
}
// Read back by asking each tool, not by trusting apt's exit code.
for _, t := range []string{"tofu", "ansible-playbook", "psql", "runuser"} {
if _, err := exec.LookPath(t); err != nil {
return fmt.Errorf("%s is still not installed", t)
}
}
if _, err := e.run(ctx, "", "ansible-doc", "-t", "module", "ansible.posix.firewalld"); err != nil {
return fmt.Errorf("ansible.posix is not available; the head node roles need it")
}
tv, _ := e.run(ctx, "", "tofu", "version")
av, _ := e.run(ctx, "", "ansible", "--version")
e.Logf("%s; %s", firstLine(tv), firstLine(av))
return nil
}
// stepAccount creates the service account and the directories it owns.
func stepAccount(ctx context.Context, e *Env) error {
if _, err := user.Lookup(ServiceID); err != nil {
if _, err := e.run(ctx, "", "useradd", "--system", "--home-dir", VarDir, "--no-create-home",
"--shell", "/usr/sbin/nologin", ServiceID); err != nil {
return err
}
e.Logf("created the %s account", ServiceID)
}
if _, err := user.Lookup(ServiceID); err != nil {
return fmt.Errorf("the %s account still does not exist", ServiceID)
}
if err := e.mkdir(EtcDir, 0o750, rootGroup); err != nil {
return err
}
// ReadWritePaths on a path that does not exist is not ignored: systemd
// refuses to start the unit. The .ansible paths are the ones
// ansible.cfg names, because the account has no real home.
for _, d := range []string{VarDir, VarDir + "/clusters", VarDir + "/terraform",
VarDir + "/terraform/envs", TemplateDir, VarDir + "/.ssh"} {
mode := os.FileMode(0o750)
if strings.HasSuffix(d, "/.ssh") {
mode = 0o700
}
if err := e.mkdir(d, mode, owner); err != nil {
return err
}
}
for _, d := range []string{VarDir + "/.ansible", VarDir + "/.ansible/tmp", VarDir + "/.ansible/cp"} {
if err := e.mkdir(d, 0o755, owner); err != nil {
return err
}
}
return nil
}
// stepBinary installs this binary as the one the unit runs.
func stepBinary(_ context.Context, e *Env) error {
if e.Exe == "" {
return fmt.Errorf("cannot tell where this binary is")
}
self, err := os.ReadFile(e.Exe)
if err != nil {
return err
}
if cur, err := os.ReadFile(e.P(BinPath)); err == nil && bytes.Equal(cur, self) {
e.Logf("%s is already this binary", BinPath)
return nil
}
if err := e.writeFile(BinPath, self, 0o755, "root:root"); err != nil {
return err
}
sum := sha256.Sum256(self)
e.Logf("installed %s (sha256 %s)", BinPath, hex.EncodeToString(sum[:6]))
return nil
}
// stepKeys makes the provisioner's SSH key and the control plane's client
// certificate, once each.
func stepKeys(_ context.Context, e *Env) error {
// The SSH key Proxmox's root and every head's CI_USER accept.
if !exists(e.P(KeyPath)) {
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
return err
}
blk, err := ssh.MarshalPrivateKey(priv, "hpcod-provisioner")
if err != nil {
return err
}
spub, err := ssh.NewPublicKey(pub)
if err != nil {
return err
}
line := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(spub))) + " hpcod-provisioner\n"
if err := e.writeFile(KeyPath, pem.EncodeToMemory(blk), 0o600, owner); err != nil {
return err
}
if err := e.writeFile(KeyPath+".pub", []byte(line), 0o644, owner); err != nil {
return err
}
e.Logf("new provisioner SSH key")
}
if _, err := e.Signer(); err != nil {
return fmt.Errorf("the provisioner key does not load: %w", err)
}
// The certificate every cluster's portal pins as its only client.
// Self-signed on purpose: nothing but the clusters this control plane
// builds needs to trust it, and they are told it directly.
crt, key := EtcDir+"/proxy-client.crt", EtcDir+"/proxy-client.key"
if !exists(e.P(crt)) {
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return err
}
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120))
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: "hpcod-control-plane"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &k.PublicKey, k)
if err != nil {
return err
}
kder, err := x509.MarshalPKCS8PrivateKey(k)
if err != nil {
return err
}
if err := e.writeFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: kder}), 0o640, rootGroup); err != nil {
return err
}
if err := e.writeFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644, rootGroup); err != nil {
return err
}
e.Logf("new control plane certificate")
}
raw, err := os.ReadFile(e.P(crt))
if err != nil {
return err
}
blk, _ := pem.Decode(raw)
if blk == nil {
return fmt.Errorf("%s is not PEM", crt)
}
c, err := x509.ParseCertificate(blk.Bytes)
if err != nil {
return err
}
if time.Until(c.NotAfter) < 30*24*time.Hour {
e.Logf("!! %s expires %s", crt, c.NotAfter.Format("2006-01-02"))
}
return nil
}
// Signer loads the provisioner key for SSH.
func (e *Env) Signer() (ssh.Signer, error) {
raw, err := os.ReadFile(e.P(KeyPath))
if err != nil {
return nil, err
}
return ssh.ParsePrivateKey(raw)
}
// PublicKey is the provisioner key's authorized_keys line.
func (e *Env) PublicKey() (string, error) {
raw, err := os.ReadFile(e.P(KeyPath + ".pub"))
return strings.TrimSpace(string(raw)), err
}
// psql runs SQL as the database owner over the local socket.
func (e *Env) psql(ctx context.Context, db, sql string, args ...string) (string, error) {
a := []string{"-u", "postgres", "--", "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1"}
if db != "" {
a = append(a, "-d", db)
}
a = append(a, args...)
return e.run(ctx, sql, "runuser", a...)
}
// stepDatabase creates the role and database on the local PostgreSQL,
// unless db.env already names one. The tables belong to postgres; the
// service role is granted rows only, so the portal cannot drop what it
// records into.
func stepDatabase(ctx context.Context, e *Env) error {
dbenv := EtcDir + "/db.env"
if raw, err := os.ReadFile(e.P(dbenv)); err == nil && !bytes.Contains(raw, []byte("CHANGE_ME")) {
e.Logf("%s already names a database; leaving it", dbenv)
return nil
}
_, _ = e.run(ctx, "", "systemctl", "enable", "--now", "postgresql")
if _, err := e.psql(ctx, "", "", "-c", "select 1"); err != nil {
return fmt.Errorf("the local PostgreSQL is not answering: %w", err)
}
pw := randomWord(32)
verb := "CREATE"
if out, _ := e.psql(ctx, "", "", "-Atc", "select 1 from pg_roles where rolname='hpcod'"); out == "1" {
verb = "ALTER"
}
// On stdin, not the command line, where every process could read it.
if _, err := e.psql(ctx, "", fmt.Sprintf("%s ROLE hpcod WITH LOGIN PASSWORD '%s';\n", verb, pw)); err != nil {
return err
}
if out, _ := e.psql(ctx, "", "", "-Atc", "select 1 from pg_database where datname='hpcod'"); out != "1" {
if _, err := e.psql(ctx, "", "", "-c", "CREATE DATABASE hpcod"); err != nil {
return err
}
}
dsn := "HPCOD_DB_DSN=postgres://hpcod:" + pw + "@127.0.0.1:5432/hpcod\n"
if err := e.writeFile(dbenv, []byte(dsn), 0o640, rootGroup); err != nil {
return err
}
e.Logf("role and database hpcod; DSN in %s", dbenv)
return nil
}
// stepConfig writes the configuration the portal and Terraform read.
// What is generated is rewritten every time, so it cannot drift from the
// site; what an operator or setup wrote once is left alone.
func stepConfig(_ context.Context, e *Env) error {
s := e.Site
// The site catalogue is seeded once and then belongs to the operator: a
// second site is added by editing it, and regenerating would drop it.
sites := EtcDir + "/sites.yaml"
if !exists(e.P(sites)) {
y := fmt.Sprintf(`sites:
- id: %s
name: %q
kind: proxmox
settings:
endpoint: %s
token: env:%s
insecure_tls: "%s"
timeout: 20s
cpu_overcommit: "1.0"
`, s["PVE_SITE_ID"], s["PVE_SITE_NAME"], s["PVE_ENDPOINT"], s.TokenVar(), s["PVE_INSECURE_TLS"])
if err := e.writeFile(sites, []byte(y), 0o640, rootGroup); err != nil {
return err
}
e.Logf("wrote %s", sites)
}
// Its own site's nodes, kept in step with the answers on every run.
if raw, err := os.ReadFile(e.P(sites)); err == nil {
out, changed, err := syncSiteSettings(raw, s["PVE_SITE_ID"], map[string]string{
"nodes": strings.Join(s.Nodes(), ","), "head_node": s["PVE_NODE"],
})
switch {
case err != nil:
// A hand-edited file setup cannot follow: said, not fatal, since
// a site without these settings still works on one node.
e.Logf("WARNING: %s: could not set the nodes of site %s (%v); add nodes: and head_node: to its settings by hand",
sites, s["PVE_SITE_ID"], err)
case changed:
if err := e.writeFile(sites, out, 0o640, rootGroup); err != nil {
return err
}
back, err := os.ReadFile(e.P(sites))
if err != nil {
return err
}
if _, again, err := syncSiteSettings(back, s["PVE_SITE_ID"], map[string]string{
"nodes": strings.Join(s.Nodes(), ","), "head_node": s["PVE_NODE"],
}); err != nil || again {
return fmt.Errorf("%s: the nodes of site %s did not stick", sites, s["PVE_SITE_ID"])
}
e.Logf("site %s: nodes %s, heads on %s", s["PVE_SITE_ID"], strings.Join(s.Nodes(), ","), s["PVE_NODE"])
}
}
// The discovery token's file. Until the Proxmox step fills it, the
// portal starts and says the site is unreachable, which is the truth.
envf := EtcDir + "/env"
if !exists(e.P(envf)) {
if err := e.writeFile(envf, []byte(s.TokenVar()+"=CHANGE_ME\n"), 0o640, rootGroup); err != nil {
return err
}
}
// Terraform's variables for this site. The token is kept apart, in
// secrets.auto.tfvars, so this file can be regenerated freely. An
// older layout kept it inline here: moved out once, before the file is
// replaced, or this would delete the only copy.
site := TemplateDir + "/site.auto.tfvars"
secrets := TemplateDir + "/secrets.auto.tfvars"
if old, err := os.ReadFile(e.P(site)); err == nil && !exists(e.P(secrets)) {
if line := tfvarLine(string(old), "pve_token"); line != "" {
if err := e.writeFile(secrets, []byte(line+"\n"), 0o640, owner); err != nil {
return err
}
bak := site + ".bak-" + time.Now().Format("20060102-150405")
if err := e.writeFile(bak, old, 0o640, owner); err != nil {
return err
}
e.Logf("moved pve_token into %s (old file kept as %s)", secrets, bak)
}
}
pub, err := e.PublicKey()
if err != nil {
return err
}
if err := e.writeFile(site, []byte(SiteTFVars(s, pub)), 0o640, owner); err != nil {
return err
}
// The portal's first login.
users := EtcDir + "/users"
if !exists(e.P(users)) {
pw := randomWord(20)
hash, err := bcrypt.GenerateFromPassword([]byte(pw), 12)
if err != nil {
return err
}
if err := e.writeFile(users, []byte("admin:"+string(hash)+"\n"), 0o640, rootGroup); err != nil {
return err
}
if err := e.writeFile("/root/.hpcod-portal-login", []byte("UI_USER=admin\nUI_PASS="+pw+"\n"), 0o600, "root:root"); err != nil {
return err
}
e.Logf("portal login: admin / %s (also in /root/.hpcod-portal-login)", pw)
}
return nil
}
// SiteTFVars is site.auto.tfvars for a site and the provisioner's key.
func SiteTFVars(s Site, provisionerKey string) string {
keys := append([]string{provisionerKey}, s.ExtraKeys()...)
var b strings.Builder
fmt.Fprintf(&b, "# Generated by `hpcod setup` from %s. Edit that and run setup again;\n", SiteFile)
b.WriteString("# changes made here are overwritten. The token is in secrets.auto.tfvars.\n")
fmt.Fprintf(&b, "pve_endpoint = %q\n", s["PVE_ENDPOINT"])
fmt.Fprintf(&b, "pve_insecure = %s\n", s["PVE_INSECURE_TLS"])
b.WriteString("pve_ssh_user = \"root\"\n")
fmt.Fprintf(&b, "pve_ssh_host = %q\n", s["PVE_HOST"])
b.WriteString("pve_ssh_port = 22\n\n")
fmt.Fprintf(&b, "node_name = %q\n", s["PVE_NODE"])
fmt.Fprintf(&b, "datastore_id = %q\n", s["PVE_DATASTORE"])
fmt.Fprintf(&b, "snippet_datastore_id = %q\n", s["PVE_SNIPPET_DATASTORE"])
fmt.Fprintf(&b, "sdn_zone = %q\n", s["PVE_SDN_ZONE"])
fmt.Fprintf(&b, "sdn_zone_type = %q\n", s.ZoneType())
fmt.Fprintf(&b, "head_template_vmid = %s\n\n", s["HEAD_TEMPLATE_VMID"])
fmt.Fprintf(&b, "mgmt_bridge = %q\n", s["MGMT_BRIDGE"])
fmt.Fprintf(&b, "mgmt_gateway = %q\n\n", s["MGMT_GATEWAY"])
fmt.Fprintf(&b, "ci_user = %q\n", s["CI_USER"])
b.WriteString("ssh_public_keys = [\n")
for i, k := range keys {
sep := ","
if i == len(keys)-1 {
sep = ""
}
fmt.Fprintf(&b, " %q%s\n", k, sep)
}
b.WriteString("]\n")
return b.String()
}
// UnitFile is the portal's systemd unit for a site.
func UnitFile(s Site) string {
return fmt.Sprintf(`# Generated by `+"`hpcod setup`"+` from %[1]s. Edit that and run setup again.
[Unit]
Description=HPC-on-Demand portal (discovery, placement, and cluster allocation)
After=network-online.target postgresql.service
Wants=network-online.target
[Service]
Type=simple
User=%[2]s
Group=%[2]s
EnvironmentFile=%[3]s/env
# The control plane database. Separate file because it holds a password.
EnvironmentFile=%[3]s/db.env
WorkingDirectory=%[3]s
ExecStart=%[4]s serve -listen 127.0.0.1:9880 \
-c %[3]s/sites.yaml -users %[3]s/users -trusted-proxy 127.0.0.1 \
-apps %[3]s/apps.yaml \
-clusters %[5]s/clusters.yaml \
-cluster-certs %[5]s/clusters \
-cluster-domain %[6]s \
-tf-envs %[5]s/terraform/envs \
-tf-template %[7]s \
-mgmt-first %[8]s -mgmt-count %[9]s \
-ssh-key %[10]s -ssh-user %[11]s \
-portal-domain %[12]s%[13]s \
-portal-cert %[3]s/proxy-client.crt -portal-key %[3]s/proxy-client.key
Restart=on-failure
RestartSec=3
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectSystem=strict
ReadWritePaths=%[5]s
ProtectHome=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
# AF_UNIX as well as the two IP families: the database is reached over a
# local socket when the DSN names one, and without it the portal starts and
# then cannot connect - which reads as a database problem rather than a
# sandbox one.
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
RestrictNamespaces=yes
LockPersonality=yes
MemoryDenyWriteExecute=yes
SystemCallArchitectures=native
%[14]s
[Install]
WantedBy=multi-user.target
`, SiteFile, ServiceID, EtcDir, BinPath, VarDir, s["CLUSTER_DOMAIN"], TemplateDir,
s["MGMT_FIRST"], s["MGMT_COUNT"], KeyPath, s["CI_USER"], s["PORTAL_DOMAIN"],
serveFlags(s), capabilities(s))
}
// serveFlags are the unit's flags for serving on a closed network.
func serveFlags(s Site) string {
var b strings.Builder
ip := s["PUBLIC_IP"]
if s.Direct() {
fmt.Fprintf(&b, " \\\n -tls-listen %s:443 -http-listen %s:80 \\\n -tls-ca %s -tls-ca-key %s",
ip, ip, CACert, CAKey)
}
if s.DNS() {
fmt.Fprintf(&b, " \\\n -dns-listen %s:53 -dns-address %s -dns-forward %s",
ip, ip, strings.Join(strings.Fields(s["NAMESERVERS"]), ","))
}
return b.String()
}
// capabilities is the unit's capability lines. None at all behind a proxy;
// binding 443, 80 and 53 needs one, and only that one.
func capabilities(s Site) string {
if s.Direct() || s.DNS() {
return "# Ports below 1024, and nothing else.\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE"
}
return "CapabilityBoundingSet="
}
// stepUnit writes the unit and reads it back through systemd. A value that
// expanded to nothing gives a valid file and a portal that starts quietly
// less than it was; systemd's own view is what runs.
func stepUnit(ctx context.Context, e *Env) error {
if err := e.writeFile(UnitPath, []byte(UnitFile(e.Site)), 0o644, "root:root"); err != nil {
return err
}
if _, err := e.run(ctx, "", "systemctl", "daemon-reload"); err != nil {
return err
}
if _, err := e.run(ctx, "", "systemctl", "enable", Unit); err != nil {
return err
}
ex, err := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ExecStart", "--value")
if err != nil {
return err
}
for _, want := range []string{
"-cluster-domain " + e.Site["CLUSTER_DOMAIN"] + " ",
"-mgmt-first " + e.Site["MGMT_FIRST"] + " ",
"-ssh-user " + e.Site["CI_USER"] + " ",
"-tf-template " + TemplateDir + " ",
"-portal-cert " + EtcDir + "/proxy-client.crt ",
"-portal-domain " + e.Site["PORTAL_DOMAIN"] + " ",
} {
if !strings.Contains(ex, want) {
return fmt.Errorf("systemd's ExecStart does not carry %q", strings.TrimSpace(want))
}
}
if e.Site.Direct() && !strings.Contains(ex, "-tls-listen "+e.Site["PUBLIC_IP"]+":443 ") {
return fmt.Errorf("systemd's ExecStart does not serve HTTPS on %s:443", e.Site["PUBLIC_IP"])
}
if e.Site.DNS() && !strings.Contains(ex, "-dns-listen "+e.Site["PUBLIC_IP"]+":53 ") {
return fmt.Errorf("systemd's ExecStart does not answer DNS on %s:53", e.Site["PUBLIC_IP"])
}
rw, _ := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ReadWritePaths", "--value")
if !strings.Contains(rw, VarDir) {
return fmt.Errorf("ReadWritePaths does not cover %s; publishing a cluster would fail", VarDir)
}
e.Logf("systemd has every flag, and %s is writable", VarDir)
return nil
}
// stepStart restarts the portal and waits until it serves. It runs one
// discovery before it listens, so this waits for an answer rather than a
// fixed time - and stops at once if the service dies.
func stepStart(ctx context.Context, e *Env) error {
if _, err := e.run(ctx, "", "systemctl", "restart", Unit); err != nil {
return err
}
for i := 0; i < 45; i++ {
if code := httpStatus(ctx, "http://127.0.0.1:9880/healthz"); code == 200 {
e.Logf("%s is serving", Unit)
return nil
}
if _, err := e.run(ctx, "", "systemctl", "is-active", "--quiet", Unit); err != nil {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(2 * time.Second):
}
}
j, _ := e.run(ctx, "", "journalctl", "-u", Unit, "-n", "20", "--no-pager")
return fmt.Errorf("%s did not come up:\n%s", Unit, j)
}
// stepCA makes the certificate authority a directly serving portal issues
// its own certificate from. Constrained to the site's domains, so trusting
// it trusts nothing else. Behind a proxy there is nothing to do.
func stepCA(_ context.Context, e *Env) error {
if !e.Site.Direct() {
e.Logf("served behind a proxy; no certificate authority needed")
return nil
}
names := e.Site.CertNames()
domains := []string{e.Site["PORTAL_DOMAIN"], e.Site["CLUSTER_DOMAIN"]}
if raw, err := os.ReadFile(e.P(CACert)); err == nil {
if ok, _ := pki.Covers(raw, domains); ok {
e.Logf("certificate authority in %s covers %s", CACert, strings.Join(pki.Zones(domains), ", "))
return verifyCA(e, names)
}
bak := CACert + ".bak-" + time.Now().Format("20060102-150405")
if err := e.writeFile(bak, raw, 0o644, rootGroup); err != nil {
return err
}
e.Logf("!! the domains changed and the old CA cannot sign for them (kept as %s).", bak)
e.Logf("!! Users must import the new one: http://%s/ca.crt", e.Site["PORTAL_DOMAIN"])
}
crt, key, err := pki.NewCA(domains)
if err != nil {
return err
}
// The portal reads the key: it issues and renews its own certificate.
if err := e.writeFile(CAKey, key, 0o640, rootGroup); err != nil {
return err
}
if err := e.writeFile(CACert, crt, 0o644, rootGroup); err != nil {
return err
}
e.Logf("new certificate authority for %s in %s", strings.Join(pki.Zones(domains), ", "), CACert)
return verifyCA(e, names)
}
// verifyCA issues a certificate the way the portal will, so a CA that
// cannot sign for the names fails here and not at the portal's start.
func verifyCA(e *Env, names []string) error {
crt, err := os.ReadFile(e.P(CACert))
if err != nil {
return err
}
key, err := os.ReadFile(e.P(CAKey))
if err != nil {
return err
}
if _, err := pki.NewIssuer(crt, key, names); err != nil {
return err
}
return nil
}
// UserInstructions is what the people using the portal need to be told,
// once, for a site that serves itself.
func UserInstructions(s Site) string {
if !s.Direct() && !s.DNS() {
return ""
}
var b strings.Builder
b.WriteString("Tell the people who will use the portal:\n")
if s.DNS() {
fmt.Fprintf(&b, " - DNS: use %s as a resolver, or forward %s to it from your own DNS.\n",
s["PUBLIC_IP"], strings.Join(pki.Zones([]string{s["PORTAL_DOMAIN"], s["CLUSTER_DOMAIN"]}), " and "))
}
if s.Direct() {
fmt.Fprintf(&b, " - Certificate: download http://%s/ca.crt and trust it, once.\n", s["PORTAL_DOMAIN"])
b.WriteString(" It can only vouch for this installation's own names.\n")
}
fmt.Fprintf(&b, " - Then: https://%s\n", s["PORTAL_DOMAIN"])
return b.String()
}
// DefaultPublicIP is this host's address on the way to the gateway: the
// one users on the management network most likely reach it at.
func DefaultPublicIP(gateway string) string {
c, err := net.Dial("udp", net.JoinHostPort(gateway, "53"))
if err != nil {
return ""
}
defer c.Close()
if a, ok := c.LocalAddr().(*net.UDPAddr); ok {
return a.IP.String()
}
return ""
}
// --- small helpers -----------------------------------------------------------
func osRelease(s, key string) string {
for _, l := range strings.Split(s, "\n") {
if k, v, ok := strings.Cut(l, "="); ok && k == key {
return strings.Trim(v, `"`)
}
}
return ""
}
func firstLine(s string) string {
l, _, _ := strings.Cut(s, "\n")
return l
}
func randomWord(n int) string {
const alpha = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"
b := make([]byte, n)
max := big.NewInt(int64(len(alpha)))
for i := range b {
v, _ := rand.Int(rand.Reader, max)
b[i] = alpha[v.Int64()]
}
return string(b)
}
var reTFLine = regexp.MustCompile(`(?m)^\s*([a-z_]+)\s*=.*$`)
// tfvarLine returns the whole line that assigns name, or "".
func tfvarLine(s, name string) string {
for _, m := range reTFLine.FindAllStringSubmatch(s, -1) {
if m[1] == name {
return strings.TrimSpace(m[0])
}
}
return ""
}
// tfvarString is the value of a `name = "value"` line.
func tfvarString(s, name string) string {
l := tfvarLine(s, name)
_, v, ok := strings.Cut(l, "=")
if !ok {
return ""
}
return strings.Trim(strings.TrimSpace(v), `"`)
}
func readAll(r io.Reader) string { b, _ := io.ReadAll(r); return string(b) }