ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
372 lines
13 KiB
Go
372 lines
13 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/pem"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"os/user"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/miekg/dns"
|
|
)
|
|
|
|
// Status of one check.
|
|
type Status string
|
|
|
|
const (
|
|
Pass Status = "PASS"
|
|
Warn Status = "WARN"
|
|
Fail Status = "FAIL"
|
|
)
|
|
|
|
// Result is one fact about the installation, read from the thing itself.
|
|
type Result struct {
|
|
Group string
|
|
Name string
|
|
Status Status
|
|
Detail string
|
|
}
|
|
|
|
// Ready reports whether nothing failed.
|
|
func Ready(rs []Result) bool {
|
|
for _, r := range rs {
|
|
if r.Status == Fail {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Verify answers "can this control plane build a cluster?" by checking
|
|
// each thing it needs by using it: a token against the API, the template
|
|
// by asking Proxmox, the database by connecting as the portal does.
|
|
// "The service is active" is one line of many, because it was true of every
|
|
// broken installation this would have caught.
|
|
func Verify(ctx context.Context, e *Env) []Result {
|
|
var rs []Result
|
|
add := func(group, name string, st Status, detail string) {
|
|
rs = append(rs, Result{group, name, st, detail})
|
|
}
|
|
check := func(group, name string, ok bool, fix string) {
|
|
if ok {
|
|
add(group, name, Pass, "")
|
|
} else {
|
|
add(group, name, Fail, fix)
|
|
}
|
|
}
|
|
s := e.Site
|
|
|
|
g := "host"
|
|
for _, t := range []string{"tofu", "ansible-playbook", "psql"} {
|
|
_, err := exec.LookPath(t)
|
|
check(g, t+" is installed", err == nil, "run setup (packages)")
|
|
}
|
|
_, err := user.Lookup(ServiceID)
|
|
check(g, "the "+ServiceID+" account exists", err == nil, "run setup (account)")
|
|
_, err = e.Signer()
|
|
check(g, "the provisioner key loads", err == nil, "run setup (keys)")
|
|
if na, err := certExpiry(e.P(EtcDir + "/proxy-client.crt")); err != nil {
|
|
add(g, "control plane certificate", Fail, err.Error())
|
|
} else if time.Until(na) < 30*24*time.Hour {
|
|
add(g, "control plane certificate", Warn, "expires "+na.Format("2006-01-02"))
|
|
} else {
|
|
add(g, "control plane certificate", Pass, "until "+na.Format("2006-01-02"))
|
|
}
|
|
|
|
g = "portal"
|
|
_, err = e.run(ctx, "", "systemctl", "is-active", "--quiet", Unit)
|
|
check(g, Unit+" is active", err == nil, "journalctl -u "+Unit)
|
|
code := httpStatus(ctx, "http://127.0.0.1:9880/healthz")
|
|
check(g, "/healthz answers 200", code == 200, fmt.Sprintf("got %d", code))
|
|
ex, _ := e.run(ctx, "", "systemctl", "show", Unit, "-p", "ExecStart", "--value")
|
|
check(g, "archives /home as "+s["CI_USER"], strings.Contains(ex, "-ssh-user "+s["CI_USER"]+" "),
|
|
"the unit is older than the site: run setup")
|
|
check(g, "routes *."+s["CLUSTER_DOMAIN"], strings.Contains(ex, "-cluster-domain "+s["CLUSTER_DOMAIN"]+" "),
|
|
"the unit is older than the site: run setup")
|
|
if raw, err := os.ReadFile(e.P(EtcDir + "/apps.yaml")); err == nil {
|
|
add(g, "application catalogue", Pass, fmt.Sprintf("%d entries", strings.Count(string(raw), "\n- name:")))
|
|
} else {
|
|
add(g, "application catalogue", Warn, "none: no applications will be offered")
|
|
}
|
|
// After the portal's own lines, so each group is listed once.
|
|
if s.Direct() {
|
|
rs = append(rs, checkTLS(ctx, e)...)
|
|
}
|
|
if s.DNS() {
|
|
rs = append(rs, checkDNS(s)...)
|
|
}
|
|
|
|
g = "database"
|
|
dsn := envValue(e.P(EtcDir+"/db.env"), "HPCOD_DB_DSN")
|
|
if dsn == "" || strings.Contains(dsn, "CHANGE_ME") {
|
|
add(g, "db.env names a database", Fail, "run setup (database)")
|
|
} else {
|
|
// As the portal connects: its role, its DSN.
|
|
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
conn, err := pgx.Connect(cctx, dsn)
|
|
if err != nil {
|
|
add(g, "the portal's role connects", Fail, err.Error())
|
|
} else {
|
|
var n int
|
|
err = conn.QueryRow(cctx, "select count(*) from cluster").Scan(&n)
|
|
check(g, "the portal's role reads clusters", err == nil, "run setup (schema)")
|
|
conn.Close(cctx)
|
|
}
|
|
cancel()
|
|
}
|
|
|
|
g = "proxmox"
|
|
if s.Sim() {
|
|
_, err := e.run(ctx, "", "systemctl", "is-active", "--quiet", SimUnit)
|
|
check(g, "simulated Proxmox ("+SimUnit+") is running", err == nil, "systemctl status "+SimUnit)
|
|
}
|
|
// One quick look first. Every check below has its own timeout, and an
|
|
// unreachable API made this section a minute of silence ending in five
|
|
// failures that all meant the same thing.
|
|
if !reachable(s["PVE_ENDPOINT"]) {
|
|
add(g, "Proxmox API reachable", Fail, "nothing answers at "+s["PVE_ENDPOINT"]+" from this host")
|
|
return append(rs, buildInputs(e)...)
|
|
}
|
|
check(g, "discovery token ("+s.TokenVar()+") accepted",
|
|
tokenWorks(ctx, s, envValue(e.P(EtcDir+"/env"), s.TokenVar())), "run setup (proxmox)")
|
|
tf := ""
|
|
if raw, err := os.ReadFile(e.P(secretsTFVars)); err == nil {
|
|
tf = tfvarString(string(raw), "pve_token")
|
|
}
|
|
check(g, "Terraform token accepted", tokenWorks(ctx, s, tf), "run setup (proxmox)")
|
|
if tokenWorks(ctx, s, tf) {
|
|
rs = append(rs, checkNodesAndZone(ctx, PVEWithToken(s["PVE_ENDPOINT"], s["PVE_INSECURE_TLS"] == "true", tf), s)...)
|
|
}
|
|
if c, err := e.PVESSH(ctx); err != nil {
|
|
add(g, "root SSH to "+s["PVE_HOST"], Fail, "run setup (proxmox) with root's password")
|
|
} else {
|
|
add(g, "root SSH to "+s["PVE_HOST"], Pass, "")
|
|
out, _ := sshRun(c, "pvesh get /storage/"+s["PVE_SNIPPET_DATASTORE"]+" --output-format json")
|
|
check(g, s["PVE_SNIPPET_DATASTORE"]+" holds snippets", strings.Contains(out, "snippets"), "run setup (proxmox)")
|
|
c.Close()
|
|
t, _ := e.Template(ctx)
|
|
switch {
|
|
case !t.Exists:
|
|
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Fail, "run setup (template)")
|
|
case !t.Current():
|
|
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Warn,
|
|
fmt.Sprintf("built from %q, roles here are %q: setup --rebuild-template", t.Baked, t.Deployed))
|
|
default:
|
|
add(g, "head template "+s["HEAD_TEMPLATE_VMID"], Pass, "built from "+t.Baked)
|
|
}
|
|
}
|
|
return append(rs, buildInputs(e)...)
|
|
}
|
|
|
|
// checkNodesAndZone checks that every node the site uses is up, and that
|
|
// the SDN zone is the type those nodes need, with each of them a peer.
|
|
func checkNodesAndZone(ctx context.Context, p *PVE, s Site) []Result {
|
|
const g = "proxmox"
|
|
nodes := s.Nodes()
|
|
label := "node " + nodes[0]
|
|
if len(nodes) > 1 {
|
|
label = "nodes " + strings.Join(nodes, ", ")
|
|
}
|
|
ips, err := nodeAddresses(ctx, p, nodes)
|
|
if err != nil {
|
|
return []Result{{g, label + " online", Fail, err.Error()}}
|
|
}
|
|
rs := []Result{{g, label + " online", Pass, ""}}
|
|
zone := "SDN zone " + s["PVE_SDN_ZONE"]
|
|
want, err := wantZone(s, ips)
|
|
if err != nil {
|
|
return append(rs, Result{g, zone, Fail, err.Error()})
|
|
}
|
|
have, err := readZone(ctx, p, s["PVE_SDN_ZONE"])
|
|
switch {
|
|
case err != nil:
|
|
rs = append(rs, Result{g, zone, Fail, err.Error()})
|
|
case have == nil:
|
|
rs = append(rs, Result{g, zone + " exists", Fail, "run setup (proxmox)"})
|
|
case !have.matches(want):
|
|
rs = append(rs, Result{g, zone + " is " + want.Type + peersNote(want), Fail,
|
|
fmt.Sprintf("it is %s%s: run setup (proxmox)", have.Type, peersNote(*have))})
|
|
default:
|
|
rs = append(rs, Result{g, zone + " is " + have.Type, Pass, strings.TrimPrefix(peersNote(*have), ", ")})
|
|
}
|
|
return rs
|
|
}
|
|
|
|
// buildInputs checks what a cluster is built from, and the names it is
|
|
// reached by. Neither needs Proxmox, so they are reported even when it
|
|
// cannot be reached.
|
|
func buildInputs(e *Env) []Result {
|
|
var rs []Result
|
|
s := e.Site
|
|
check := func(group, name string, ok bool, fix string) {
|
|
if ok {
|
|
rs = append(rs, Result{group, name, Pass, ""})
|
|
} else {
|
|
rs = append(rs, Result{group, name, Fail, fix})
|
|
}
|
|
}
|
|
|
|
g := "build inputs"
|
|
for _, f := range []string{"main.tf", "variables.tf", "site.auto.tfvars", "secrets.auto.tfvars"} {
|
|
check(g, "template/"+f, exists(e.P(TemplateDir+"/"+f)), "run setup")
|
|
}
|
|
raw, _ := os.ReadFile(e.P(TemplateDir + "/site.auto.tfvars"))
|
|
check(g, "heads are built as "+s["CI_USER"], tfvarString(string(raw), "ci_user") == s["CI_USER"], "run setup (config)")
|
|
check(g, "the tenant module is deployed", exists(e.P(VarDir+"/terraform/modules/tenant/main.tf")), "run setup (tree)")
|
|
got, _ := os.ReadFile(e.P(VarDir + "/ansible/group_vars/all/site.yml"))
|
|
check(g, "the roles carry this site's values", string(got) == SiteYAML(s), "run setup (tree)")
|
|
|
|
g = "names"
|
|
if s.DNS() {
|
|
// This host's own resolver does not point at itself; what matters
|
|
// is what this host's DNS server says, checked above.
|
|
return rs
|
|
}
|
|
if a := lookup(s["PORTAL_DOMAIN"]); a != "" {
|
|
rs = append(rs, Result{g, s["PORTAL_DOMAIN"] + " resolves", Pass, a})
|
|
} else {
|
|
rs = append(rs, Result{g, s["PORTAL_DOMAIN"] + " resolves", Warn, "no DNS record"})
|
|
}
|
|
if a := lookup("c999." + s["CLUSTER_DOMAIN"]); a != "" {
|
|
rs = append(rs, Result{g, "*." + s["CLUSTER_DOMAIN"] + " resolves", Pass, a})
|
|
} else {
|
|
rs = append(rs, Result{g, "*." + s["CLUSTER_DOMAIN"] + " resolves", Warn, "no wildcard record: cluster portals will not be reachable"})
|
|
}
|
|
return rs
|
|
}
|
|
|
|
// checkTLS connects as a browser that trusts this installation's CA would,
|
|
// for the portal and for a cluster's name.
|
|
func checkTLS(ctx context.Context, e *Env) []Result {
|
|
s := e.Site
|
|
g := "serving"
|
|
raw, err := os.ReadFile(e.P(CACert))
|
|
if err != nil {
|
|
return []Result{{g, "certificate authority", Fail, "no " + CACert + ": run setup (ca)"}}
|
|
}
|
|
pool := x509.NewCertPool()
|
|
pool.AppendCertsFromPEM(raw)
|
|
var rs []Result
|
|
for _, host := range []string{s["PORTAL_DOMAIN"], "c999." + s["CLUSTER_DOMAIN"]} {
|
|
d := &tls.Dialer{NetDialer: &net.Dialer{Timeout: 5 * time.Second},
|
|
Config: &tls.Config{ServerName: host, RootCAs: pool, MinVersion: tls.VersionTLS12}}
|
|
c, err := d.DialContext(ctx, "tcp", net.JoinHostPort(s["PUBLIC_IP"], "443"))
|
|
if err != nil {
|
|
rs = append(rs, Result{g, "HTTPS for " + host, Fail, err.Error()})
|
|
continue
|
|
}
|
|
na := c.(*tls.Conn).ConnectionState().PeerCertificates[0].NotAfter
|
|
c.Close()
|
|
rs = append(rs, Result{g, "HTTPS for " + host, Pass, "trusted by the CA, until " + na.Format("2006-01-02")})
|
|
}
|
|
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+s["PUBLIC_IP"]+"/ca.crt", nil)
|
|
req.Host = s["PORTAL_DOMAIN"]
|
|
cl := &http.Client{Timeout: 5 * time.Second}
|
|
if resp, err := cl.Do(req); err != nil || resp.StatusCode != 200 {
|
|
detail := fmt.Sprint(err)
|
|
if err == nil {
|
|
detail = resp.Status
|
|
resp.Body.Close()
|
|
}
|
|
rs = append(rs, Result{g, "CA downloadable at http://" + s["PORTAL_DOMAIN"] + "/ca.crt", Fail, detail})
|
|
} else {
|
|
resp.Body.Close()
|
|
rs = append(rs, Result{g, "CA downloadable at http://" + s["PORTAL_DOMAIN"] + "/ca.crt", Pass, ""})
|
|
}
|
|
return rs
|
|
}
|
|
|
|
// checkDNS asks this host's DNS server what a user's resolver would.
|
|
func checkDNS(s Site) []Result {
|
|
g := "serving"
|
|
server := net.JoinHostPort(s["PUBLIC_IP"], "53")
|
|
var rs []Result
|
|
for _, host := range []string{s["PORTAL_DOMAIN"], "c999." + s["CLUSTER_DOMAIN"]} {
|
|
got, err := queryA(server, host)
|
|
switch {
|
|
case err != nil:
|
|
rs = append(rs, Result{g, "DNS answers " + host, Fail, err.Error()})
|
|
case got != s["PUBLIC_IP"]:
|
|
rs = append(rs, Result{g, "DNS answers " + host, Fail, "says " + got + ", not " + s["PUBLIC_IP"]})
|
|
default:
|
|
rs = append(rs, Result{g, "DNS answers " + host, Pass, got})
|
|
}
|
|
}
|
|
if got, err := queryA(server, "deb.debian.org"); err != nil || got == "" {
|
|
rs = append(rs, Result{g, "DNS forwards other names", Warn, fmt.Sprintf("%v: users who use it as their resolver reach nothing else", err)})
|
|
} else {
|
|
rs = append(rs, Result{g, "DNS forwards other names", Pass, "deb.debian.org -> " + got})
|
|
}
|
|
return rs
|
|
}
|
|
|
|
func queryA(server, name string) (string, error) {
|
|
m := new(dns.Msg)
|
|
m.SetQuestion(dns.Fqdn(name), dns.TypeA)
|
|
m.RecursionDesired = true
|
|
in, _, err := (&dns.Client{Timeout: 3 * time.Second}).Exchange(m, server)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
for _, rr := range in.Answer {
|
|
if a, ok := rr.(*dns.A); ok {
|
|
return a.A.String(), nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("no A record (%s)", dns.RcodeToString[in.Rcode])
|
|
}
|
|
|
|
// reachable reports whether anything accepts a TCP connection at the
|
|
// endpoint's host and port within three seconds.
|
|
func reachable(endpoint string) bool {
|
|
u, err := url.Parse(endpoint)
|
|
if err != nil || u.Host == "" {
|
|
return false
|
|
}
|
|
host := u.Host
|
|
if u.Port() == "" {
|
|
host = net.JoinHostPort(u.Hostname(), "443")
|
|
}
|
|
c, err := net.DialTimeout("tcp", host, 3*time.Second)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
c.Close()
|
|
return true
|
|
}
|
|
|
|
func certExpiry(p string) (time.Time, error) {
|
|
raw, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
b, _ := pem.Decode(raw)
|
|
if b == nil {
|
|
return time.Time{}, fmt.Errorf("%s is not PEM", p)
|
|
}
|
|
c, err := x509.ParseCertificate(b.Bytes)
|
|
if err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
return c.NotAfter, nil
|
|
}
|
|
|
|
func lookup(host string) string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
a, err := net.DefaultResolver.LookupHost(ctx, host)
|
|
if err != nil || len(a) == 0 {
|
|
return ""
|
|
}
|
|
return a[0]
|
|
}
|