ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
772 lines
26 KiB
Go
772 lines
26 KiB
Go
package setup
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/ed25519"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/hex"
|
|
"encoding/pem"
|
|
"fmt"
|
|
"io"
|
|
"math/big"
|
|
"net"
|
|
"os"
|
|
"os/exec"
|
|
"os/user"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/bcrypt"
|
|
"golang.org/x/crypto/ssh"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/pki"
|
|
)
|
|
|
|
const owner = ServiceID + ":" + ServiceID
|
|
|
|
// A mirror that drops one request out of a few hundred fails a whole
|
|
// install without this; apt's default is not to retry at all.
|
|
const aptRetries = "Acquire::Retries=3"
|
|
const rootGroup = "root:" + ServiceID
|
|
|
|
// stepSite records the site where every later step - and every later run
|
|
// of this binary - reads it from.
|
|
func stepSite(_ context.Context, e *Env) error {
|
|
e.Site.ApplySim()
|
|
if errs := e.Site.Validate(); len(errs) > 0 {
|
|
var b strings.Builder
|
|
for _, err := range errs {
|
|
b.WriteString("\n " + err.Error())
|
|
}
|
|
return fmt.Errorf("the site is not complete:%s", b.String())
|
|
}
|
|
// Owned by root alone for now: on a first install the hpcod group does
|
|
// not exist until the account step, which gives the directory to it.
|
|
if err := e.mkdir(EtcDir, 0o750, ""); err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(SiteFile, []byte(e.Site.Render()), 0o644, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("site written to %s", SiteFile)
|
|
return nil
|
|
}
|
|
|
|
// stepPackages installs what the control plane drives: tofu, ansible,
|
|
// PostgreSQL. Everything else is in this binary.
|
|
func stepPackages(ctx context.Context, e *Env) error {
|
|
osr, err := os.ReadFile("/etc/os-release")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
id := osRelease(string(osr), "ID") + " " + osRelease(string(osr), "ID_LIKE")
|
|
if !strings.Contains(" "+id+" ", " debian ") && !strings.Contains(" "+id+" ", " ubuntu ") {
|
|
return fmt.Errorf("this installs with apt, and %s is not Debian or Ubuntu.\n"+
|
|
"Install postgresql, ansible and tofu by hand, then run setup with --skip packages",
|
|
osRelease(string(osr), "PRETTY_NAME"))
|
|
}
|
|
env := []string{"DEBIAN_FRONTEND=noninteractive"}
|
|
|
|
// ansible, not ansible-core: the roles use ansible.posix, which only the
|
|
// full package carries.
|
|
want := []string{"postgresql", "ansible", "openssh-client", "python3-yaml", "curl", "gnupg", "ca-certificates"}
|
|
if e.Site.Sim() {
|
|
want = append(want, "qemu-system-x86", "qemu-utils", "genisoimage", "nftables", "iproute2")
|
|
}
|
|
var need []string
|
|
for _, p := range want {
|
|
if out, _ := e.run(ctx, "", "dpkg-query", "-W", "-f=${Status}", p); !strings.Contains(out, "ok installed") {
|
|
need = append(need, p)
|
|
}
|
|
}
|
|
if len(need) > 0 {
|
|
e.Logf("installing %s", strings.Join(need, " "))
|
|
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "update", "-qq"); err != nil {
|
|
return err
|
|
}
|
|
args := append([]string{"-o", aptRetries, "install", "-y", "-qq", "--no-install-recommends"}, need...)
|
|
if err := e.stream(ctx, env, "", "apt-get", args...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// OpenTofu from its own repository; distributions do not package it.
|
|
if _, err := exec.LookPath("tofu"); err != nil {
|
|
e.Logf("adding the OpenTofu repository")
|
|
if err := e.mkdir("/etc/apt/keyrings", 0o755, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
for _, k := range []struct{ url, path string }{
|
|
{"https://get.opentofu.org/opentofu.gpg", "/etc/apt/keyrings/opentofu.gpg"},
|
|
{"https://packages.opentofu.org/opentofu/tofu/gpgkey", "/etc/apt/keyrings/opentofu-repo.gpg"},
|
|
} {
|
|
raw, err := e.run(ctx, "", "sh", "-c",
|
|
fmt.Sprintf("curl -fsSL %s | gpg --no-tty --batch --yes --dearmor -o %s && chmod a+r %s", k.url, k.path, k.path))
|
|
if err != nil {
|
|
return fmt.Errorf("fetching %s: %w %s", k.url, err, raw)
|
|
}
|
|
}
|
|
list := "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] " +
|
|
"https://packages.opentofu.org/opentofu/tofu/any/ any main\n"
|
|
if err := e.writeFile("/etc/apt/sources.list.d/opentofu.list", []byte(list), 0o644, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "update", "-qq"); err != nil {
|
|
return err
|
|
}
|
|
if err := e.stream(ctx, env, "", "apt-get", "-o", aptRetries, "install", "-y", "-qq", "tofu"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Read back by asking each tool, not by trusting apt's exit code.
|
|
for _, t := range []string{"tofu", "ansible-playbook", "psql", "runuser"} {
|
|
if _, err := exec.LookPath(t); err != nil {
|
|
return fmt.Errorf("%s is still not installed", t)
|
|
}
|
|
}
|
|
if _, err := e.run(ctx, "", "ansible-doc", "-t", "module", "ansible.posix.firewalld"); err != nil {
|
|
return fmt.Errorf("ansible.posix is not available; the head node roles need it")
|
|
}
|
|
tv, _ := e.run(ctx, "", "tofu", "version")
|
|
av, _ := e.run(ctx, "", "ansible", "--version")
|
|
e.Logf("%s; %s", firstLine(tv), firstLine(av))
|
|
return nil
|
|
}
|
|
|
|
// stepAccount creates the service account and the directories it owns.
|
|
func stepAccount(ctx context.Context, e *Env) error {
|
|
if _, err := user.Lookup(ServiceID); err != nil {
|
|
if _, err := e.run(ctx, "", "useradd", "--system", "--home-dir", VarDir, "--no-create-home",
|
|
"--shell", "/usr/sbin/nologin", ServiceID); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("created the %s account", ServiceID)
|
|
}
|
|
if _, err := user.Lookup(ServiceID); err != nil {
|
|
return fmt.Errorf("the %s account still does not exist", ServiceID)
|
|
}
|
|
if err := e.mkdir(EtcDir, 0o750, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
// ReadWritePaths on a path that does not exist is not ignored: systemd
|
|
// refuses to start the unit. The .ansible paths are the ones
|
|
// ansible.cfg names, because the account has no real home.
|
|
for _, d := range []string{VarDir, VarDir + "/clusters", VarDir + "/terraform",
|
|
VarDir + "/terraform/envs", TemplateDir, VarDir + "/.ssh"} {
|
|
mode := os.FileMode(0o750)
|
|
if strings.HasSuffix(d, "/.ssh") {
|
|
mode = 0o700
|
|
}
|
|
if err := e.mkdir(d, mode, owner); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, d := range []string{VarDir + "/.ansible", VarDir + "/.ansible/tmp", VarDir + "/.ansible/cp"} {
|
|
if err := e.mkdir(d, 0o755, owner); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// stepBinary installs this binary as the one the unit runs.
|
|
func stepBinary(_ context.Context, e *Env) error {
|
|
if e.Exe == "" {
|
|
return fmt.Errorf("cannot tell where this binary is")
|
|
}
|
|
self, err := os.ReadFile(e.Exe)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if cur, err := os.ReadFile(e.P(BinPath)); err == nil && bytes.Equal(cur, self) {
|
|
e.Logf("%s is already this binary", BinPath)
|
|
return nil
|
|
}
|
|
if err := e.writeFile(BinPath, self, 0o755, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
sum := sha256.Sum256(self)
|
|
e.Logf("installed %s (sha256 %s)", BinPath, hex.EncodeToString(sum[:6]))
|
|
return nil
|
|
}
|
|
|
|
// stepKeys makes the provisioner's SSH key and the control plane's client
|
|
// certificate, once each.
|
|
func stepKeys(_ context.Context, e *Env) error {
|
|
// The SSH key Proxmox's root and every head's CI_USER accept.
|
|
if !exists(e.P(KeyPath)) {
|
|
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
blk, err := ssh.MarshalPrivateKey(priv, "hpcod-provisioner")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
spub, err := ssh.NewPublicKey(pub)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
line := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(spub))) + " hpcod-provisioner\n"
|
|
if err := e.writeFile(KeyPath, pem.EncodeToMemory(blk), 0o600, owner); err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(KeyPath+".pub", []byte(line), 0o644, owner); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("new provisioner SSH key")
|
|
}
|
|
if _, err := e.Signer(); err != nil {
|
|
return fmt.Errorf("the provisioner key does not load: %w", err)
|
|
}
|
|
|
|
// The certificate every cluster's portal pins as its only client.
|
|
// Self-signed on purpose: nothing but the clusters this control plane
|
|
// builds needs to trust it, and they are told it directly.
|
|
crt, key := EtcDir+"/proxy-client.crt", EtcDir+"/proxy-client.key"
|
|
if !exists(e.P(crt)) {
|
|
k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serial, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 120))
|
|
tmpl := &x509.Certificate{
|
|
SerialNumber: serial,
|
|
Subject: pkix.Name{CommonName: "hpcod-control-plane"},
|
|
NotBefore: time.Now().Add(-time.Hour),
|
|
NotAfter: time.Now().AddDate(10, 0, 0),
|
|
KeyUsage: x509.KeyUsageDigitalSignature,
|
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
}
|
|
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &k.PublicKey, k)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kder, err := x509.MarshalPKCS8PrivateKey(k)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: kder}), 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("new control plane certificate")
|
|
}
|
|
raw, err := os.ReadFile(e.P(crt))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
blk, _ := pem.Decode(raw)
|
|
if blk == nil {
|
|
return fmt.Errorf("%s is not PEM", crt)
|
|
}
|
|
c, err := x509.ParseCertificate(blk.Bytes)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if time.Until(c.NotAfter) < 30*24*time.Hour {
|
|
e.Logf("!! %s expires %s", crt, c.NotAfter.Format("2006-01-02"))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Signer loads the provisioner key for SSH.
|
|
func (e *Env) Signer() (ssh.Signer, error) {
|
|
raw, err := os.ReadFile(e.P(KeyPath))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return ssh.ParsePrivateKey(raw)
|
|
}
|
|
|
|
// PublicKey is the provisioner key's authorized_keys line.
|
|
func (e *Env) PublicKey() (string, error) {
|
|
raw, err := os.ReadFile(e.P(KeyPath + ".pub"))
|
|
return strings.TrimSpace(string(raw)), err
|
|
}
|
|
|
|
// psql runs SQL as the database owner over the local socket.
|
|
func (e *Env) psql(ctx context.Context, db, sql string, args ...string) (string, error) {
|
|
a := []string{"-u", "postgres", "--", "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1"}
|
|
if db != "" {
|
|
a = append(a, "-d", db)
|
|
}
|
|
a = append(a, args...)
|
|
return e.run(ctx, sql, "runuser", a...)
|
|
}
|
|
|
|
// stepDatabase creates the role and database on the local PostgreSQL,
|
|
// unless db.env already names one. The tables belong to postgres; the
|
|
// service role is granted rows only, so the portal cannot drop what it
|
|
// records into.
|
|
func stepDatabase(ctx context.Context, e *Env) error {
|
|
dbenv := EtcDir + "/db.env"
|
|
if raw, err := os.ReadFile(e.P(dbenv)); err == nil && !bytes.Contains(raw, []byte("CHANGE_ME")) {
|
|
e.Logf("%s already names a database; leaving it", dbenv)
|
|
return nil
|
|
}
|
|
_, _ = e.run(ctx, "", "systemctl", "enable", "--now", "postgresql")
|
|
if _, err := e.psql(ctx, "", "", "-c", "select 1"); err != nil {
|
|
return fmt.Errorf("the local PostgreSQL is not answering: %w", err)
|
|
}
|
|
pw := randomWord(32)
|
|
verb := "CREATE"
|
|
if out, _ := e.psql(ctx, "", "", "-Atc", "select 1 from pg_roles where rolname='hpcod'"); out == "1" {
|
|
verb = "ALTER"
|
|
}
|
|
// On stdin, not the command line, where every process could read it.
|
|
if _, err := e.psql(ctx, "", fmt.Sprintf("%s ROLE hpcod WITH LOGIN PASSWORD '%s';\n", verb, pw)); err != nil {
|
|
return err
|
|
}
|
|
if out, _ := e.psql(ctx, "", "", "-Atc", "select 1 from pg_database where datname='hpcod'"); out != "1" {
|
|
if _, err := e.psql(ctx, "", "", "-c", "CREATE DATABASE hpcod"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
dsn := "HPCOD_DB_DSN=postgres://hpcod:" + pw + "@127.0.0.1:5432/hpcod\n"
|
|
if err := e.writeFile(dbenv, []byte(dsn), 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("role and database hpcod; DSN in %s", dbenv)
|
|
return nil
|
|
}
|
|
|
|
// stepConfig writes the configuration the portal and Terraform read.
|
|
// What is generated is rewritten every time, so it cannot drift from the
|
|
// site; what an operator or setup wrote once is left alone.
|
|
func stepConfig(_ context.Context, e *Env) error {
|
|
s := e.Site
|
|
|
|
// The site catalogue is seeded once and then belongs to the operator: a
|
|
// second site is added by editing it, and regenerating would drop it.
|
|
sites := EtcDir + "/sites.yaml"
|
|
if !exists(e.P(sites)) {
|
|
y := fmt.Sprintf(`sites:
|
|
- id: %s
|
|
name: %q
|
|
kind: proxmox
|
|
settings:
|
|
endpoint: %s
|
|
token: env:%s
|
|
insecure_tls: "%s"
|
|
timeout: 20s
|
|
cpu_overcommit: "1.0"
|
|
`, s["PVE_SITE_ID"], s["PVE_SITE_NAME"], s["PVE_ENDPOINT"], s.TokenVar(), s["PVE_INSECURE_TLS"])
|
|
if err := e.writeFile(sites, []byte(y), 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("wrote %s", sites)
|
|
}
|
|
// Its own site's nodes, kept in step with the answers on every run.
|
|
if raw, err := os.ReadFile(e.P(sites)); err == nil {
|
|
out, changed, err := syncSiteSettings(raw, s["PVE_SITE_ID"], map[string]string{
|
|
"nodes": strings.Join(s.Nodes(), ","), "head_node": s["PVE_NODE"],
|
|
})
|
|
switch {
|
|
case err != nil:
|
|
// A hand-edited file setup cannot follow: said, not fatal, since
|
|
// a site without these settings still works on one node.
|
|
e.Logf("WARNING: %s: could not set the nodes of site %s (%v); add nodes: and head_node: to its settings by hand",
|
|
sites, s["PVE_SITE_ID"], err)
|
|
case changed:
|
|
if err := e.writeFile(sites, out, 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
back, err := os.ReadFile(e.P(sites))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, again, err := syncSiteSettings(back, s["PVE_SITE_ID"], map[string]string{
|
|
"nodes": strings.Join(s.Nodes(), ","), "head_node": s["PVE_NODE"],
|
|
}); err != nil || again {
|
|
return fmt.Errorf("%s: the nodes of site %s did not stick", sites, s["PVE_SITE_ID"])
|
|
}
|
|
e.Logf("site %s: nodes %s, heads on %s", s["PVE_SITE_ID"], strings.Join(s.Nodes(), ","), s["PVE_NODE"])
|
|
}
|
|
}
|
|
|
|
// The discovery token's file. Until the Proxmox step fills it, the
|
|
// portal starts and says the site is unreachable, which is the truth.
|
|
envf := EtcDir + "/env"
|
|
if !exists(e.P(envf)) {
|
|
if err := e.writeFile(envf, []byte(s.TokenVar()+"=CHANGE_ME\n"), 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Terraform's variables for this site. The token is kept apart, in
|
|
// secrets.auto.tfvars, so this file can be regenerated freely. An
|
|
// older layout kept it inline here: moved out once, before the file is
|
|
// replaced, or this would delete the only copy.
|
|
site := TemplateDir + "/site.auto.tfvars"
|
|
secrets := TemplateDir + "/secrets.auto.tfvars"
|
|
if old, err := os.ReadFile(e.P(site)); err == nil && !exists(e.P(secrets)) {
|
|
if line := tfvarLine(string(old), "pve_token"); line != "" {
|
|
if err := e.writeFile(secrets, []byte(line+"\n"), 0o640, owner); err != nil {
|
|
return err
|
|
}
|
|
bak := site + ".bak-" + time.Now().Format("20060102-150405")
|
|
if err := e.writeFile(bak, old, 0o640, owner); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("moved pve_token into %s (old file kept as %s)", secrets, bak)
|
|
}
|
|
}
|
|
pub, err := e.PublicKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(site, []byte(SiteTFVars(s, pub)), 0o640, owner); err != nil {
|
|
return err
|
|
}
|
|
|
|
// The portal's first login.
|
|
users := EtcDir + "/users"
|
|
if !exists(e.P(users)) {
|
|
pw := randomWord(20)
|
|
hash, err := bcrypt.GenerateFromPassword([]byte(pw), 12)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(users, []byte("admin:"+string(hash)+"\n"), 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile("/root/.hpcod-portal-login", []byte("UI_USER=admin\nUI_PASS="+pw+"\n"), 0o600, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("portal login: admin / %s (also in /root/.hpcod-portal-login)", pw)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SiteTFVars is site.auto.tfvars for a site and the provisioner's key.
|
|
func SiteTFVars(s Site, provisionerKey string) string {
|
|
keys := append([]string{provisionerKey}, s.ExtraKeys()...)
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "# Generated by `hpcod setup` from %s. Edit that and run setup again;\n", SiteFile)
|
|
b.WriteString("# changes made here are overwritten. The token is in secrets.auto.tfvars.\n")
|
|
fmt.Fprintf(&b, "pve_endpoint = %q\n", s["PVE_ENDPOINT"])
|
|
fmt.Fprintf(&b, "pve_insecure = %s\n", s["PVE_INSECURE_TLS"])
|
|
b.WriteString("pve_ssh_user = \"root\"\n")
|
|
fmt.Fprintf(&b, "pve_ssh_host = %q\n", s["PVE_HOST"])
|
|
b.WriteString("pve_ssh_port = 22\n\n")
|
|
fmt.Fprintf(&b, "node_name = %q\n", s["PVE_NODE"])
|
|
fmt.Fprintf(&b, "datastore_id = %q\n", s["PVE_DATASTORE"])
|
|
fmt.Fprintf(&b, "snippet_datastore_id = %q\n", s["PVE_SNIPPET_DATASTORE"])
|
|
fmt.Fprintf(&b, "sdn_zone = %q\n", s["PVE_SDN_ZONE"])
|
|
fmt.Fprintf(&b, "sdn_zone_type = %q\n", s.ZoneType())
|
|
fmt.Fprintf(&b, "head_template_vmid = %s\n\n", s["HEAD_TEMPLATE_VMID"])
|
|
fmt.Fprintf(&b, "mgmt_bridge = %q\n", s["MGMT_BRIDGE"])
|
|
fmt.Fprintf(&b, "mgmt_gateway = %q\n\n", s["MGMT_GATEWAY"])
|
|
fmt.Fprintf(&b, "ci_user = %q\n", s["CI_USER"])
|
|
b.WriteString("ssh_public_keys = [\n")
|
|
for i, k := range keys {
|
|
sep := ","
|
|
if i == len(keys)-1 {
|
|
sep = ""
|
|
}
|
|
fmt.Fprintf(&b, " %q%s\n", k, sep)
|
|
}
|
|
b.WriteString("]\n")
|
|
return b.String()
|
|
}
|
|
|
|
// UnitFile is the portal's systemd unit for a site.
|
|
func UnitFile(s Site) string {
|
|
return fmt.Sprintf(`# Generated by `+"`hpcod setup`"+` from %[1]s. Edit that and run setup again.
|
|
[Unit]
|
|
Description=HPC-on-Demand portal (discovery, placement, and cluster allocation)
|
|
After=network-online.target postgresql.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=%[2]s
|
|
Group=%[2]s
|
|
EnvironmentFile=%[3]s/env
|
|
# The control plane database. Separate file because it holds a password.
|
|
EnvironmentFile=%[3]s/db.env
|
|
WorkingDirectory=%[3]s
|
|
ExecStart=%[4]s serve -listen 127.0.0.1:9880 \
|
|
-c %[3]s/sites.yaml -users %[3]s/users -trusted-proxy 127.0.0.1 \
|
|
-apps %[3]s/apps.yaml \
|
|
-clusters %[5]s/clusters.yaml \
|
|
-cluster-certs %[5]s/clusters \
|
|
-cluster-domain %[6]s \
|
|
-tf-envs %[5]s/terraform/envs \
|
|
-tf-template %[7]s \
|
|
-mgmt-first %[8]s -mgmt-count %[9]s \
|
|
-ssh-key %[10]s -ssh-user %[11]s \
|
|
-portal-domain %[12]s%[13]s \
|
|
-portal-cert %[3]s/proxy-client.crt -portal-key %[3]s/proxy-client.key
|
|
Restart=on-failure
|
|
RestartSec=3
|
|
|
|
NoNewPrivileges=yes
|
|
PrivateTmp=yes
|
|
PrivateDevices=yes
|
|
ProtectSystem=strict
|
|
ReadWritePaths=%[5]s
|
|
ProtectHome=yes
|
|
ProtectKernelTunables=yes
|
|
ProtectKernelModules=yes
|
|
ProtectControlGroups=yes
|
|
# AF_UNIX as well as the two IP families: the database is reached over a
|
|
# local socket when the DSN names one, and without it the portal starts and
|
|
# then cannot connect - which reads as a database problem rather than a
|
|
# sandbox one.
|
|
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
|
RestrictNamespaces=yes
|
|
LockPersonality=yes
|
|
MemoryDenyWriteExecute=yes
|
|
SystemCallArchitectures=native
|
|
%[14]s
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
`, SiteFile, ServiceID, EtcDir, BinPath, VarDir, s["CLUSTER_DOMAIN"], TemplateDir,
|
|
s["MGMT_FIRST"], s["MGMT_COUNT"], KeyPath, s["CI_USER"], s["PORTAL_DOMAIN"],
|
|
serveFlags(s), capabilities(s))
|
|
}
|
|
|
|
// serveFlags are the unit's flags for serving on a closed network.
|
|
func serveFlags(s Site) string {
|
|
var b strings.Builder
|
|
ip := s["PUBLIC_IP"]
|
|
if s.Direct() {
|
|
fmt.Fprintf(&b, " \\\n -tls-listen %s:443 -http-listen %s:80 \\\n -tls-ca %s -tls-ca-key %s",
|
|
ip, ip, CACert, CAKey)
|
|
}
|
|
if s.DNS() {
|
|
fmt.Fprintf(&b, " \\\n -dns-listen %s:53 -dns-address %s -dns-forward %s",
|
|
ip, ip, strings.Join(strings.Fields(s["NAMESERVERS"]), ","))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// capabilities is the unit's capability lines. None at all behind a proxy;
|
|
// binding 443, 80 and 53 needs one, and only that one.
|
|
func capabilities(s Site) string {
|
|
if s.Direct() || s.DNS() {
|
|
return "# Ports below 1024, and nothing else.\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE"
|
|
}
|
|
return "CapabilityBoundingSet="
|
|
}
|
|
|
|
// stepUnit writes the unit and reads it back through systemd. A value that
|
|
// expanded to nothing gives a valid file and a portal that starts quietly
|
|
// less than it was; systemd's own view is what runs.
|
|
func stepUnit(ctx context.Context, e *Env) error {
|
|
if err := e.writeFile(UnitPath, []byte(UnitFile(e.Site)), 0o644, "root:root"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "daemon-reload"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "enable", Unit); err != nil {
|
|
return err
|
|
}
|
|
ex, err := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ExecStart", "--value")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, want := range []string{
|
|
"-cluster-domain " + e.Site["CLUSTER_DOMAIN"] + " ",
|
|
"-mgmt-first " + e.Site["MGMT_FIRST"] + " ",
|
|
"-ssh-user " + e.Site["CI_USER"] + " ",
|
|
"-tf-template " + TemplateDir + " ",
|
|
"-portal-cert " + EtcDir + "/proxy-client.crt ",
|
|
"-portal-domain " + e.Site["PORTAL_DOMAIN"] + " ",
|
|
} {
|
|
if !strings.Contains(ex, want) {
|
|
return fmt.Errorf("systemd's ExecStart does not carry %q", strings.TrimSpace(want))
|
|
}
|
|
}
|
|
if e.Site.Direct() && !strings.Contains(ex, "-tls-listen "+e.Site["PUBLIC_IP"]+":443 ") {
|
|
return fmt.Errorf("systemd's ExecStart does not serve HTTPS on %s:443", e.Site["PUBLIC_IP"])
|
|
}
|
|
if e.Site.DNS() && !strings.Contains(ex, "-dns-listen "+e.Site["PUBLIC_IP"]+":53 ") {
|
|
return fmt.Errorf("systemd's ExecStart does not answer DNS on %s:53", e.Site["PUBLIC_IP"])
|
|
}
|
|
rw, _ := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ReadWritePaths", "--value")
|
|
if !strings.Contains(rw, VarDir) {
|
|
return fmt.Errorf("ReadWritePaths does not cover %s; publishing a cluster would fail", VarDir)
|
|
}
|
|
e.Logf("systemd has every flag, and %s is writable", VarDir)
|
|
return nil
|
|
}
|
|
|
|
// stepStart restarts the portal and waits until it serves. It runs one
|
|
// discovery before it listens, so this waits for an answer rather than a
|
|
// fixed time - and stops at once if the service dies.
|
|
func stepStart(ctx context.Context, e *Env) error {
|
|
if _, err := e.run(ctx, "", "systemctl", "restart", Unit); err != nil {
|
|
return err
|
|
}
|
|
for i := 0; i < 45; i++ {
|
|
if code := httpStatus(ctx, "http://127.0.0.1:9880/healthz"); code == 200 {
|
|
e.Logf("%s is serving", Unit)
|
|
return nil
|
|
}
|
|
if _, err := e.run(ctx, "", "systemctl", "is-active", "--quiet", Unit); err != nil {
|
|
break
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(2 * time.Second):
|
|
}
|
|
}
|
|
j, _ := e.run(ctx, "", "journalctl", "-u", Unit, "-n", "20", "--no-pager")
|
|
return fmt.Errorf("%s did not come up:\n%s", Unit, j)
|
|
}
|
|
|
|
// stepCA makes the certificate authority a directly serving portal issues
|
|
// its own certificate from. Constrained to the site's domains, so trusting
|
|
// it trusts nothing else. Behind a proxy there is nothing to do.
|
|
func stepCA(_ context.Context, e *Env) error {
|
|
if !e.Site.Direct() {
|
|
e.Logf("served behind a proxy; no certificate authority needed")
|
|
return nil
|
|
}
|
|
names := e.Site.CertNames()
|
|
domains := []string{e.Site["PORTAL_DOMAIN"], e.Site["CLUSTER_DOMAIN"]}
|
|
if raw, err := os.ReadFile(e.P(CACert)); err == nil {
|
|
if ok, _ := pki.Covers(raw, domains); ok {
|
|
e.Logf("certificate authority in %s covers %s", CACert, strings.Join(pki.Zones(domains), ", "))
|
|
return verifyCA(e, names)
|
|
}
|
|
bak := CACert + ".bak-" + time.Now().Format("20060102-150405")
|
|
if err := e.writeFile(bak, raw, 0o644, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("!! the domains changed and the old CA cannot sign for them (kept as %s).", bak)
|
|
e.Logf("!! Users must import the new one: http://%s/ca.crt", e.Site["PORTAL_DOMAIN"])
|
|
}
|
|
crt, key, err := pki.NewCA(domains)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// The portal reads the key: it issues and renews its own certificate.
|
|
if err := e.writeFile(CAKey, key, 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(CACert, crt, 0o644, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("new certificate authority for %s in %s", strings.Join(pki.Zones(domains), ", "), CACert)
|
|
return verifyCA(e, names)
|
|
}
|
|
|
|
// verifyCA issues a certificate the way the portal will, so a CA that
|
|
// cannot sign for the names fails here and not at the portal's start.
|
|
func verifyCA(e *Env, names []string) error {
|
|
crt, err := os.ReadFile(e.P(CACert))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
key, err := os.ReadFile(e.P(CAKey))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := pki.NewIssuer(crt, key, names); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UserInstructions is what the people using the portal need to be told,
|
|
// once, for a site that serves itself.
|
|
func UserInstructions(s Site) string {
|
|
if !s.Direct() && !s.DNS() {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
b.WriteString("Tell the people who will use the portal:\n")
|
|
if s.DNS() {
|
|
fmt.Fprintf(&b, " - DNS: use %s as a resolver, or forward %s to it from your own DNS.\n",
|
|
s["PUBLIC_IP"], strings.Join(pki.Zones([]string{s["PORTAL_DOMAIN"], s["CLUSTER_DOMAIN"]}), " and "))
|
|
}
|
|
if s.Direct() {
|
|
fmt.Fprintf(&b, " - Certificate: download http://%s/ca.crt and trust it, once.\n", s["PORTAL_DOMAIN"])
|
|
b.WriteString(" It can only vouch for this installation's own names.\n")
|
|
}
|
|
fmt.Fprintf(&b, " - Then: https://%s\n", s["PORTAL_DOMAIN"])
|
|
return b.String()
|
|
}
|
|
|
|
// DefaultPublicIP is this host's address on the way to the gateway: the
|
|
// one users on the management network most likely reach it at.
|
|
func DefaultPublicIP(gateway string) string {
|
|
c, err := net.Dial("udp", net.JoinHostPort(gateway, "53"))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer c.Close()
|
|
if a, ok := c.LocalAddr().(*net.UDPAddr); ok {
|
|
return a.IP.String()
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// --- small helpers -----------------------------------------------------------
|
|
|
|
func osRelease(s, key string) string {
|
|
for _, l := range strings.Split(s, "\n") {
|
|
if k, v, ok := strings.Cut(l, "="); ok && k == key {
|
|
return strings.Trim(v, `"`)
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
l, _, _ := strings.Cut(s, "\n")
|
|
return l
|
|
}
|
|
|
|
func randomWord(n int) string {
|
|
const alpha = "ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz23456789"
|
|
b := make([]byte, n)
|
|
max := big.NewInt(int64(len(alpha)))
|
|
for i := range b {
|
|
v, _ := rand.Int(rand.Reader, max)
|
|
b[i] = alpha[v.Int64()]
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
var reTFLine = regexp.MustCompile(`(?m)^\s*([a-z_]+)\s*=.*$`)
|
|
|
|
// tfvarLine returns the whole line that assigns name, or "".
|
|
func tfvarLine(s, name string) string {
|
|
for _, m := range reTFLine.FindAllStringSubmatch(s, -1) {
|
|
if m[1] == name {
|
|
return strings.TrimSpace(m[0])
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// tfvarString is the value of a `name = "value"` line.
|
|
func tfvarString(s, name string) string {
|
|
l := tfvarLine(s, name)
|
|
_, v, ok := strings.Cut(l, "=")
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return strings.Trim(strings.TrimSpace(v), `"`)
|
|
}
|
|
|
|
func readAll(r io.Reader) string { b, _ := io.ReadAll(r); return string(b) }
|