Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

497 lines
16 KiB
Go

// Package setup installs and maintains a control plane from the binary
// alone: the host, the database, the tree clusters are built from,
// Proxmox, the head node template, and a check of all of it.
//
// It replaces a set of shell scripts (B05, B10, B15, B20, B99, C20) that had
// to be copied to the host beside the binary and kept in step with it. Each
// step here does one thing and then reads back the state it changed - the
// exit code of a command says it ran, not that anything is different.
package setup
import (
"bufio"
"fmt"
"net"
"net/url"
"os"
"regexp"
"strconv"
"strings"
)
// Field is one value that differs between installations. The list below is
// the only description of a site: the form asks for these, site.env stores
// them, and Validate checks them.
type Field struct {
Key string
Label string
Help string
Default string
Group string
// Optional fields may be empty.
Optional bool
// RequiredIf makes an optional field required when the rest of the
// site needs it.
requiredIf func(s Site) bool
check func(v string, s Site) error
}
// Site holds a site's values by Field.Key.
type Site map[string]string
const (
groupNames = "Names"
groupProxmox = "Proxmox"
groupNetwork = "Management network"
groupHeads = "Head nodes and images"
groupAccess = "How users reach the portal"
)
// Fields is ordered as the form presents it.
var Fields = []Field{
{Key: "PORTAL_DOMAIN", Group: groupNames, Label: "Portal domain",
Help: "Where the portal answers, e.g. hpc.example.org.", check: hostname},
{Key: "CLUSTER_DOMAIN", Group: groupNames, Label: "Cluster domain",
Help: "Each cluster's portal is c<NNN>.<this>; needs a wildcard DNS record. Usually the same as the portal domain.",
check: hostname},
{Key: "SERVE_MODE", Group: groupAccess, Label: "Serving",
Help: "proxy: behind your own reverse proxy and certificate. direct: hpcod serves HTTPS itself, from a CA of its own - for a closed network with no public domain.",
Default: "proxy", check: oneOf("proxy", "direct")},
{Key: "SERVE_DNS", Group: groupAccess, Label: "Answer DNS for these names",
Help: "true runs a DNS server for the portal and *.cluster names; point users' resolvers (or a forwarder for the zone) at PUBLIC_IP.",
Default: "false", check: boolean},
{Key: "PUBLIC_IP", Group: groupAccess, Label: "Address users reach this host at",
Help: "Where HTTPS and DNS listen, and what the names resolve to. Needed for direct serving or DNS.",
Optional: true, requiredIf: func(s Site) bool { return s.Direct() || s.DNS() }, check: ipv4},
{Key: "PVE_MODE", Group: groupProxmox, Label: "Proxmox to build clusters on",
Help: "real: a Proxmox VE you have. sim: a simulated one, in QEMU on this host, to try hpcod - setup fills in every Proxmox and network value below.",
Default: "real", check: oneOf("real", "sim")},
{Key: "SIM_CORES", Group: groupProxmox, Label: "Simulated Proxmox: cores", Default: "8", check: positive},
{Key: "SIM_MEMORY_GB", Group: groupProxmox, Label: "Simulated Proxmox: memory (GiB)", Default: "24", check: positive},
{Key: "SIM_DISK_GB", Group: groupProxmox, Label: "Simulated Proxmox: disk (GiB)", Default: "150", check: positive},
{Key: "PVE_ENDPOINT", Group: groupProxmox, Label: "Proxmox API",
Help: "e.g. https://192.0.2.10:8006", check: endpoint},
{Key: "PVE_INSECURE_TLS", Group: groupProxmox, Label: "Accept a self-signed certificate",
Help: "true while the API presents Proxmox's own certificate.", Default: "true", check: boolean},
{Key: "PVE_HOST", Group: groupProxmox, Label: "Proxmox SSH address",
Help: "Where this host reaches the hypervisor as root over SSH. Usually the API's host.", check: hostOrIP},
{Key: "PVE_NODE", Group: groupProxmox, Label: "Proxmox node",
Help: "The node the template, the snippets and every head node live on.", check: name},
{Key: "PVE_NODES", Group: groupProxmox, Label: "Nodes clusters may use",
Help: "Proxmox nodes compute nodes may be placed on, separated by commas. Empty means PVE_NODE alone. More than one needs a vxlan SDN zone.",
Optional: true, check: nodeList},
{Key: "PVE_SITE_ID", Group: groupProxmox, Label: "Site id",
Help: "What the portal calls this Proxmox; names its token variable.", Default: "hv01", check: name},
{Key: "PVE_SITE_NAME", Group: groupProxmox, Label: "Site name",
Help: "Shown on the portal's pages.", Default: "Proxmox"},
{Key: "PVE_DATASTORE", Group: groupProxmox, Label: "Datastore for VM disks", Default: "local-zfs", check: name},
{Key: "PVE_SNIPPET_DATASTORE", Group: groupProxmox, Label: "Datastore for cloud-init snippets",
Help: "Setup enables the snippets content type on it.", Default: "local", check: name},
{Key: "PVE_IMAGE", Group: groupProxmox, Label: "Cloud image",
Help: "storage:import/file. Downloaded if missing.",
Default: "local:import/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2", check: volume},
{Key: "PVE_SDN_ZONE", Group: groupProxmox, Label: "SDN zone",
Help: "Created if missing; every cluster gets a vnet in it.", Default: "hpcod", check: sdnZone},
{Key: "PVE_SDN_ZONE_TYPE", Group: groupProxmox, Label: "SDN zone type",
Help: "auto: simple on one node, vxlan across several. simple gives each node its own bridge, so it cannot carry a cluster across nodes; vxlan can.",
Default: "auto", check: zoneType},
{Key: "MGMT_BRIDGE", Group: groupNetwork, Label: "Management bridge", Default: "vmbr0", check: name},
{Key: "MGMT_GATEWAY", Group: groupNetwork, Label: "Gateway", Help: "Only /24 networks are supported.", check: ipv4},
{Key: "MGMT_FIRST", Group: groupNetwork, Label: "First head node address",
Help: "A run of free addresses outside DHCP, one per live cluster.", check: inGatewayNet},
{Key: "MGMT_COUNT", Group: groupNetwork, Label: "How many addresses", Default: "40", check: mgmtCount},
{Key: "BUILD_IP", Group: groupNetwork, Label: "Template build address",
Help: "One more free address, for the VM the head template is built in.", check: buildIP},
{Key: "NAMESERVERS", Group: groupNetwork, Label: "DNS servers", Default: "1.1.1.1 8.8.8.8", check: ipList},
{Key: "HEAD_TEMPLATE_VMID", Group: groupHeads, Label: "Head template VMID", Default: "9100", check: vmid},
{Key: "CI_USER", Group: groupHeads, Label: "Head node login account",
Help: "Created by cloud-init on every head; the portal archives /home through it.", Default: "hpcod", check: unixUser},
{Key: "EXTRA_SSH_KEYS", Group: groupHeads, Label: "Extra SSH public keys", Optional: true,
Help: "Installed for that account on every head, separated by ';'. This host's key is always included."},
{Key: "REGISTRY_HOST", Group: groupHeads, Label: "Image registry", Default: "gitea.mynodes.xyz", check: hostOrIP},
{Key: "NODE_IMAGE_URI", Group: groupHeads, Label: "Compute node image",
Default: "docker://gitea.mynodes.xyz/hpcod/node-image:rocky9-ohpc3", check: imageURI},
{Key: "DESKTOP_IMAGE", Group: groupHeads, Label: "Desktop image",
Default: "docker://gitea.mynodes.xyz/hpcapps/desktop:xfce", check: imageURI},
}
// FieldByKey finds a field's description.
func FieldByKey(k string) (Field, bool) {
for _, f := range Fields {
if f.Key == k {
return f, true
}
}
return Field{}, false
}
// Defaults is a site with every default filled in.
func Defaults() Site {
s := Site{}
for _, f := range Fields {
s[f.Key] = f.Default
}
return s
}
// Check validates one value in the context of the rest of the site.
func (f Field) Check(v string, s Site) error {
v = strings.TrimSpace(v)
if v == "" {
if f.Optional && (f.requiredIf == nil || !f.requiredIf(s)) {
return nil
}
return fmt.Errorf("required")
}
if strings.ContainsAny(v, "\"$`\\\n") {
return fmt.Errorf("may not contain quotes, $, backquotes or backslashes")
}
if f.check != nil {
return f.check(v, s)
}
return nil
}
// Validate checks every field, and returns every problem rather than the
// first: a form that reports one error per submission is a form that
// takes eight submissions.
func (s Site) Validate() []error {
var errs []error
for _, f := range Fields {
if err := f.Check(s[f.Key], s); err != nil {
errs = append(errs, fmt.Errorf("%s (%s): %w", f.Label, f.Key, err))
}
}
return errs
}
// TokenVar is the environment variable the discovery token lives in:
// hv01 -> HV01_TOKEN.
func (s Site) TokenVar() string {
b := []byte(strings.ToUpper(s["PVE_SITE_ID"]) + "_TOKEN")
for i, c := range b {
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
b[i] = '_'
}
}
return string(b)
}
// Nodes are the Proxmox nodes clusters may use, PVE_NODE first. PVE_NODE
// is always one of them: it holds the template, so every head runs there.
func (s Site) Nodes() []string {
out := []string{s["PVE_NODE"]}
for _, n := range strings.FieldsFunc(s["PVE_NODES"], func(r rune) bool { return r == ',' || r == ' ' }) {
if !contains(out, n) {
out = append(out, n)
}
}
return out
}
// ZoneType is the SDN zone type the site needs: what PVE_SDN_ZONE_TYPE
// says, with auto resolved by the number of nodes.
func (s Site) ZoneType() string {
if t := s["PVE_SDN_ZONE_TYPE"]; t == "simple" || t == "vxlan" {
return t
}
if len(s.Nodes()) > 1 {
return "vxlan"
}
return "simple"
}
func contains(list []string, v string) bool {
for _, x := range list {
if x == v {
return true
}
}
return false
}
// Direct reports whether the portal serves HTTPS itself.
func (s Site) Direct() bool { return s["SERVE_MODE"] == "direct" }
// DNS reports whether the portal answers DNS for its names.
func (s Site) DNS() bool { return s["SERVE_DNS"] == "true" }
// CertNames are the names the portal's certificate covers.
func (s Site) CertNames() []string {
out := []string{s["PORTAL_DOMAIN"]}
for _, n := range []string{s["CLUSTER_DOMAIN"], "*." + s["CLUSTER_DOMAIN"]} {
if n != out[0] {
out = append(out, n)
}
}
return out
}
// ExtraKeys splits EXTRA_SSH_KEYS.
func (s Site) ExtraKeys() []string {
var out []string
for _, k := range strings.Split(s["EXTRA_SSH_KEYS"], ";") {
if k = strings.TrimSpace(k); k != "" {
out = append(out, k)
}
}
return out
}
// LoadSite reads a site.env: KEY=value lines, optional double quotes. Keys
// it does not know are kept, so a file written by a newer binary survives
// a round trip through an older one.
func LoadSite(path string) (Site, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
s := Site{}
sc := bufio.NewScanner(f)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
k, v, ok := strings.Cut(line, "=")
if !ok {
continue
}
v = strings.TrimSpace(v)
if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' {
v = v[1 : len(v)-1]
}
s[strings.TrimSpace(k)] = v
}
return s, sc.Err()
}
// Render writes the site in site.env form, grouped and commented, so the
// file the form saves reads like the one a person would write.
func (s Site) Render() string {
var b strings.Builder
b.WriteString("# This installation's values. Written by `hpcod setup`; safe to edit by\n")
b.WriteString("# hand and re-run it. Nothing here is secret.\n")
if h := s["DEPLOY_HOST"]; h != "" {
fmt.Fprintf(&b, "\n# Where `make deploy` installs from a workstation.\nDEPLOY_HOST=%s\n", h)
}
group := ""
for _, f := range Fields {
if f.Group != group {
group = f.Group
fmt.Fprintf(&b, "\n# --- %s\n", group)
}
if f.Help != "" {
fmt.Fprintf(&b, "# %s\n", f.Help)
}
v := s[f.Key]
if strings.ContainsAny(v, " ;#") || v == "" {
v = `"` + v + `"`
}
fmt.Fprintf(&b, "%s=%s\n", f.Key, v)
}
return b.String()
}
// --- checks -----------------------------------------------------------------
var (
reName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
reHost = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$`)
reUnixUser = regexp.MustCompile(`^[a-z_][a-z0-9_-]{0,31}$`)
reSDNZone = regexp.MustCompile(`^[a-z][a-z0-9]{0,7}$`)
)
func name(v string, _ Site) error {
if !reName.MatchString(v) {
return fmt.Errorf("letters, digits, '.', '_' and '-' only")
}
return nil
}
func hostname(v string, _ Site) error {
if !reHost.MatchString(v) {
return fmt.Errorf("not a domain name")
}
return nil
}
func hostOrIP(v string, s Site) error {
if net.ParseIP(v) != nil {
return nil
}
return hostname(v, s)
}
func endpoint(v string, _ Site) error {
u, err := url.Parse(v)
if err != nil || u.Scheme != "https" || u.Host == "" || (u.Path != "" && u.Path != "/") {
return fmt.Errorf("expected https://host:8006")
}
return nil
}
func oneOf(vals ...string) func(string, Site) error {
return func(v string, _ Site) error {
for _, x := range vals {
if v == x {
return nil
}
}
return fmt.Errorf("one of: %s", strings.Join(vals, ", "))
}
}
func boolean(v string, _ Site) error {
if v != "true" && v != "false" {
return fmt.Errorf("true or false")
}
return nil
}
func volume(v string, _ Site) error {
store, file, ok := strings.Cut(v, ":import/")
if !ok || store == "" || file == "" || strings.Contains(file, "/") {
return fmt.Errorf("expected storage:import/file.qcow2")
}
return nil
}
func sdnZone(v string, _ Site) error {
// Proxmox's own rule for zone ids.
if !reSDNZone.MatchString(v) {
return fmt.Errorf("lowercase letter, then up to 7 lowercase letters or digits")
}
return nil
}
func nodeList(v string, s Site) error {
names := strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
for _, n := range names {
if err := name(n, s); err != nil {
return fmt.Errorf("%s: %w", n, err)
}
}
// Written out in full or not at all: a list that leaves out the
// template's node reads as if heads could not run there.
if len(names) > 0 && s["PVE_NODE"] != "" && !contains(names, s["PVE_NODE"]) {
return fmt.Errorf("must include %s, where the template and the heads are", s["PVE_NODE"])
}
return nil
}
func zoneType(v string, s Site) error {
if err := oneOf("auto", "simple", "vxlan")(v, s); err != nil {
return err
}
if v == "simple" && len(s.Nodes()) > 1 {
return fmt.Errorf("a simple zone is one bridge per node, so a cluster across %d nodes would be split; use vxlan or auto", len(s.Nodes()))
}
return nil
}
func ipv4(v string, _ Site) error {
ip := net.ParseIP(v)
if ip == nil || ip.To4() == nil {
return fmt.Errorf("not an IPv4 address")
}
return nil
}
func ipList(v string, s Site) error {
for _, ip := range strings.Fields(v) {
if err := ipv4(ip, s); err != nil {
return fmt.Errorf("%s: %w", ip, err)
}
}
return nil
}
// net24 is the /24 the gateway is on, which is the management network:
// the head node prefix is fixed at 24.
func net24(ip string) string {
p := net.ParseIP(ip).To4()
if p == nil {
return ""
}
return fmt.Sprintf("%d.%d.%d", p[0], p[1], p[2])
}
func lastOctet(ip string) int {
p := net.ParseIP(ip).To4()
if p == nil {
return -1
}
return int(p[3])
}
func inGatewayNet(v string, s Site) error {
if err := ipv4(v, s); err != nil {
return err
}
if g := s["MGMT_GATEWAY"]; net24(g) != "" && net24(v) != net24(g) {
return fmt.Errorf("not on the gateway's network %s.0/24", net24(g))
}
if v == s["MGMT_GATEWAY"] {
return fmt.Errorf("is the gateway")
}
return nil
}
func mgmtCount(v string, s Site) error {
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return fmt.Errorf("a positive number")
}
if first := lastOctet(s["MGMT_FIRST"]); first > 0 && first+n-1 > 254 {
return fmt.Errorf("runs past .254 from %s", s["MGMT_FIRST"])
}
return nil
}
func buildIP(v string, s Site) error {
if err := inGatewayNet(v, s); err != nil {
return err
}
first, n := lastOctet(s["MGMT_FIRST"]), 0
n, _ = strconv.Atoi(s["MGMT_COUNT"])
if o := lastOctet(v); first > 0 && net24(v) == net24(s["MGMT_FIRST"]) && o >= first && o < first+n {
return fmt.Errorf("inside the head node window; a cluster could be given it")
}
return nil
}
func positive(v string, _ Site) error {
if n, err := strconv.Atoi(v); err != nil || n < 1 {
return fmt.Errorf("a positive number")
}
return nil
}
func vmid(v string, _ Site) error {
n, err := strconv.Atoi(v)
if err != nil || n < 100 || n > 999999999 {
return fmt.Errorf("a VMID, 100 or more")
}
return nil
}
func unixUser(v string, _ Site) error {
if !reUnixUser.MatchString(v) || v == "root" {
return fmt.Errorf("a login name, not root")
}
return nil
}
func imageURI(v string, _ Site) error {
if !strings.HasPrefix(v, "docker://") && !strings.HasPrefix(v, "oras://") {
return fmt.Errorf("expected docker://… or oras://…")
}
return nil
}