Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

372 lines
13 KiB
Go

package setup
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"os/user"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/miekg/dns"
)
// Status of one check.
type Status string
const (
Pass Status = "PASS"
Warn Status = "WARN"
Fail Status = "FAIL"
)
// Result is one fact about the installation, read from the thing itself.
type Result struct {
Group string
Name string
Status Status
Detail string
}
// Ready reports whether nothing failed.
func Ready(rs []Result) bool {
for _, r := range rs {
if r.Status == Fail {
return false
}
}
return true
}
// Verify answers "can this control plane build a cluster?" by checking
// each thing it needs by using it: a token against the API, the template
// by asking Proxmox, the database by connecting as the portal does.
// "The service is active" is one line of many, because it was true of every
// broken installation this would have caught.
func Verify(ctx context.Context, e *Env) []Result {
var rs []Result
add := func(group, name string, st Status, detail string) {
rs = append(rs, Result{group, name, st, detail})
}
check := func(group, name string, ok bool, fix string) {
if ok {
add(group, name, Pass, "")
} else {
add(group, name, Fail, fix)
}
}
s := e.Site
g := "host"
for _, t := range []string{"tofu", "ansible-playbook", "psql"} {
_, err := exec.LookPath(t)
check(g, t+" is installed", err == nil, "run setup (packages)")
}
_, err := user.Lookup(ServiceID)
check(g, "the "+ServiceID+" account exists", err == nil, "run setup (account)")
_, err = e.Signer()
check(g, "the provisioner key loads", err == nil, "run setup (keys)")
if na, err := certExpiry(e.P(EtcDir + "/proxy-client.crt")); err != nil {
add(g, "control plane certificate", Fail, err.Error())
} else if time.Until(na) < 30*24*time.Hour {
add(g, "control plane certificate", Warn, "expires "+na.Format("2006-01-02"))
} else {
add(g, "control plane certificate", Pass, "until "+na.Format("2006-01-02"))
}
g = "portal"
_, err = e.run(ctx, "", "systemctl", "is-active", "--quiet", Unit)
check(g, Unit+" is active", err == nil, "journalctl -u "+Unit)
code := httpStatus(ctx, "http://127.0.0.1:9880/healthz")
check(g, "/healthz answers 200", code == 200, fmt.Sprintf("got %d", code))
ex, _ := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ExecStart", "--value")
check(g, "archives /home as "+s["CI_USER"], strings.Contains(ex, "-ssh-user "+s["CI_USER"]+" "),
"the unit is older than the site: run setup")
check(g, "routes *."+s["CLUSTER_DOMAIN"], strings.Contains(ex, "-cluster-domain "+s["CLUSTER_DOMAIN"]+" "),
"the unit is older than the site: run setup")
if raw, err := os.ReadFile(e.P(EtcDir + "/apps.yaml")); err == nil {
add(g, "application catalogue", Pass, fmt.Sprintf("%d entries", strings.Count(string(raw), "\n- name:")))
} else {
add(g, "application catalogue", Warn, "none: no applications will be offered")
}
// After the portal's own lines, so each group is listed once.
if s.Direct() {
rs = append(rs, checkTLS(ctx, e)...)
}
if s.DNS() {
rs = append(rs, checkDNS(s)...)
}
g = "database"
dsn := envValue(e.P(EtcDir+"/db.env"), "HPCOD_DB_DSN")
if dsn == "" || strings.Contains(dsn, "CHANGE_ME") {
add(g, "db.env names a database", Fail, "run setup (database)")
} else {
// As the portal connects: its role, its DSN.
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
conn, err := pgx.Connect(cctx, dsn)
if err != nil {
add(g, "the portal's role connects", Fail, err.Error())
} else {
var n int
err = conn.QueryRow(cctx, "select count(*) from cluster").Scan(&n)
check(g, "the portal's role reads clusters", err == nil, "run setup (schema)")
conn.Close(cctx)
}
cancel()
}
g = "proxmox"
if s.Sim() {
_, err := e.run(ctx, "", "systemctl", "is-active", "--quiet", SimUnit)
check(g, "simulated Proxmox ("+SimUnit+") is running", err == nil, "systemctl status "+SimUnit)
}
// One quick look first. Every check below has its own timeout, and an
// unreachable API made this section a minute of silence ending in five
// failures that all meant the same thing.
if !reachable(s["PVE_ENDPOINT"]) {
add(g, "Proxmox API reachable", Fail, "nothing answers at "+s["PVE_ENDPOINT"]+" from this host")
return append(rs, buildInputs(e)...)
}
check(g, "discovery token ("+s.TokenVar()+") accepted",
tokenWorks(ctx, s, envValue(e.P(EtcDir+"/env"), s.TokenVar())), "run setup (proxmox)")
tf := ""
if raw, err := os.ReadFile(e.P(secretsTFVars)); err == nil {
tf = tfvarString(string(raw), "pve_token")
}
check(g, "Terraform token accepted", tokenWorks(ctx, s, tf), "run setup (proxmox)")
if tokenWorks(ctx, s, tf) {
rs = append(rs, checkNodesAndZone(ctx, PVEWithToken(s["PVE_ENDPOINT"], s["PVE_INSECURE_TLS"] == "true", tf), s)...)
}
if c, err := e.PVESSH(ctx); err != nil {
add(g, "root SSH to "+s["PVE_HOST"], Fail, "run setup (proxmox) with root's password")
} else {
add(g, "root SSH to "+s["PVE_HOST"], Pass, "")
out, _ := sshRun(c, "pvesh get /storage/"+s["PVE_SNIPPET_DATASTORE"]+" --output-format json")
check(g, s["PVE_SNIPPET_DATASTORE"]+" holds snippets", strings.Contains(out, "snippets"), "run setup (proxmox)")
c.Close()
t, _ := e.Template(ctx)
switch {
case !t.Exists:
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Fail, "run setup (template)")
case !t.Current():
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Warn,
fmt.Sprintf("built from %q, roles here are %q: setup --rebuild-template", t.Baked, t.Deployed))
default:
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Pass, "built from "+t.Baked)
}
}
return append(rs, buildInputs(e)...)
}
// checkNodesAndZone checks that every node the site uses is up, and that
// the SDN zone is the type those nodes need, with each of them a peer.
func checkNodesAndZone(ctx context.Context, p *PVE, s Site) []Result {
const g = "proxmox"
nodes := s.Nodes()
label := "node " + nodes[0]
if len(nodes) > 1 {
label = "nodes " + strings.Join(nodes, ", ")
}
ips, err := nodeAddresses(ctx, p, nodes)
if err != nil {
return []Result{{g, label + " online", Fail, err.Error()}}
}
rs := []Result{{g, label + " online", Pass, ""}}
zone := "SDN zone " + s["PVE_SDN_ZONE"]
want, err := wantZone(s, ips)
if err != nil {
return append(rs, Result{g, zone, Fail, err.Error()})
}
have, err := readZone(ctx, p, s["PVE_SDN_ZONE"])
switch {
case err != nil:
rs = append(rs, Result{g, zone, Fail, err.Error()})
case have == nil:
rs = append(rs, Result{g, zone + " exists", Fail, "run setup (proxmox)"})
case !have.matches(want):
rs = append(rs, Result{g, zone + " is " + want.Type + peersNote(want), Fail,
fmt.Sprintf("it is %s%s: run setup (proxmox)", have.Type, peersNote(*have))})
default:
rs = append(rs, Result{g, zone + " is " + have.Type, Pass, strings.TrimPrefix(peersNote(*have), ", ")})
}
return rs
}
// buildInputs checks what a cluster is built from, and the names it is
// reached by. Neither needs Proxmox, so they are reported even when it
// cannot be reached.
func buildInputs(e *Env) []Result {
var rs []Result
s := e.Site
check := func(group, name string, ok bool, fix string) {
if ok {
rs = append(rs, Result{group, name, Pass, ""})
} else {
rs = append(rs, Result{group, name, Fail, fix})
}
}
g := "build inputs"
for _, f := range []string{"main.tf", "variables.tf", "site.auto.tfvars", "secrets.auto.tfvars"} {
check(g, "template/"+f, exists(e.P(TemplateDir+"/"+f)), "run setup")
}
raw, _ := os.ReadFile(e.P(TemplateDir + "/site.auto.tfvars"))
check(g, "heads are built as "+s["CI_USER"], tfvarString(string(raw), "ci_user") == s["CI_USER"], "run setup (config)")
check(g, "the tenant module is deployed", exists(e.P(VarDir+"/terraform/modules/tenant/main.tf")), "run setup (tree)")
got, _ := os.ReadFile(e.P(VarDir + "/ansible/group_vars/all/site.yml"))
check(g, "the roles carry this site's values", string(got) == SiteYAML(s), "run setup (tree)")
g = "names"
if s.DNS() {
// This host's own resolver does not point at itself; what matters
// is what this host's DNS server says, checked above.
return rs
}
if a := lookup(s["PORTAL_DOMAIN"]); a != "" {
rs = append(rs, Result{g, s["PORTAL_DOMAIN"] + " resolves", Pass, a})
} else {
rs = append(rs, Result{g, s["PORTAL_DOMAIN"] + " resolves", Warn, "no DNS record"})
}
if a := lookup("c999." + s["CLUSTER_DOMAIN"]); a != "" {
rs = append(rs, Result{g, "*." + s["CLUSTER_DOMAIN"] + " resolves", Pass, a})
} else {
rs = append(rs, Result{g, "*." + s["CLUSTER_DOMAIN"] + " resolves", Warn, "no wildcard record: cluster portals will not be reachable"})
}
return rs
}
// checkTLS connects as a browser that trusts this installation's CA would,
// for the portal and for a cluster's name.
func checkTLS(ctx context.Context, e *Env) []Result {
s := e.Site
g := "serving"
raw, err := os.ReadFile(e.P(CACert))
if err != nil {
return []Result{{g, "certificate authority", Fail, "no " + CACert + ": run setup (ca)"}}
}
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(raw)
var rs []Result
for _, host := range []string{s["PORTAL_DOMAIN"], "c999." + s["CLUSTER_DOMAIN"]} {
d := &tls.Dialer{NetDialer: &net.Dialer{Timeout: 5 * time.Second},
Config: &tls.Config{ServerName: host, RootCAs: pool, MinVersion: tls.VersionTLS12}}
c, err := d.DialContext(ctx, "tcp", net.JoinHostPort(s["PUBLIC_IP"], "443"))
if err != nil {
rs = append(rs, Result{g, "HTTPS for " + host, Fail, err.Error()})
continue
}
na := c.(*tls.Conn).ConnectionState().PeerCertificates[0].NotAfter
c.Close()
rs = append(rs, Result{g, "HTTPS for " + host, Pass, "trusted by the CA, until " + na.Format("2006-01-02")})
}
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+s["PUBLIC_IP"]+"/ca.crt", nil)
req.Host = s["PORTAL_DOMAIN"]
cl := &http.Client{Timeout: 5 * time.Second}
if resp, err := cl.Do(req); err != nil || resp.StatusCode != 200 {
detail := fmt.Sprint(err)
if err == nil {
detail = resp.Status
resp.Body.Close()
}
rs = append(rs, Result{g, "CA downloadable at http://" + s["PORTAL_DOMAIN"] + "/ca.crt", Fail, detail})
} else {
resp.Body.Close()
rs = append(rs, Result{g, "CA downloadable at http://" + s["PORTAL_DOMAIN"] + "/ca.crt", Pass, ""})
}
return rs
}
// checkDNS asks this host's DNS server what a user's resolver would.
func checkDNS(s Site) []Result {
g := "serving"
server := net.JoinHostPort(s["PUBLIC_IP"], "53")
var rs []Result
for _, host := range []string{s["PORTAL_DOMAIN"], "c999." + s["CLUSTER_DOMAIN"]} {
got, err := queryA(server, host)
switch {
case err != nil:
rs = append(rs, Result{g, "DNS answers " + host, Fail, err.Error()})
case got != s["PUBLIC_IP"]:
rs = append(rs, Result{g, "DNS answers " + host, Fail, "says " + got + ", not " + s["PUBLIC_IP"]})
default:
rs = append(rs, Result{g, "DNS answers " + host, Pass, got})
}
}
if got, err := queryA(server, "deb.debian.org"); err != nil || got == "" {
rs = append(rs, Result{g, "DNS forwards other names", Warn, fmt.Sprintf("%v: users who use it as their resolver reach nothing else", err)})
} else {
rs = append(rs, Result{g, "DNS forwards other names", Pass, "deb.debian.org -> " + got})
}
return rs
}
func queryA(server, name string) (string, error) {
m := new(dns.Msg)
m.SetQuestion(dns.Fqdn(name), dns.TypeA)
m.RecursionDesired = true
in, _, err := (&dns.Client{Timeout: 3 * time.Second}).Exchange(m, server)
if err != nil {
return "", err
}
for _, rr := range in.Answer {
if a, ok := rr.(*dns.A); ok {
return a.A.String(), nil
}
}
return "", fmt.Errorf("no A record (%s)", dns.RcodeToString[in.Rcode])
}
// reachable reports whether anything accepts a TCP connection at the
// endpoint's host and port within three seconds.
func reachable(endpoint string) bool {
u, err := url.Parse(endpoint)
if err != nil || u.Host == "" {
return false
}
host := u.Host
if u.Port() == "" {
host = net.JoinHostPort(u.Hostname(), "443")
}
c, err := net.DialTimeout("tcp", host, 3*time.Second)
if err != nil {
return false
}
c.Close()
return true
}
func certExpiry(p string) (time.Time, error) {
raw, err := os.ReadFile(p)
if err != nil {
return time.Time{}, err
}
b, _ := pem.Decode(raw)
if b == nil {
return time.Time{}, fmt.Errorf("%s is not PEM", p)
}
c, err := x509.ParseCertificate(b.Bytes)
if err != nil {
return time.Time{}, err
}
return c.NotAfter, nil
}
func lookup(host string) string {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
a, err := net.DefaultResolver.LookupHost(ctx, host)
if err != nil || len(a) == 0 {
return ""
}
return a[0]
}