ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
497 lines
16 KiB
Go
497 lines
16 KiB
Go
// Package setup installs and maintains a control plane from the binary
|
|
// alone: the host, the database, the tree clusters are built from,
|
|
// Proxmox, the head node template, and a check of all of it.
|
|
//
|
|
// It replaces a set of shell scripts (B05, B10, B15, B20, B99, C20) that had
|
|
// to be copied to the host beside the binary and kept in step with it. Each
|
|
// step here does one thing and then reads back the state it changed - the
|
|
// exit code of a command says it ran, not that anything is different.
|
|
package setup
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Field is one value that differs between installations. The list below is
|
|
// the only description of a site: the form asks for these, site.env stores
|
|
// them, and Validate checks them.
|
|
type Field struct {
|
|
Key string
|
|
Label string
|
|
Help string
|
|
Default string
|
|
Group string
|
|
// Optional fields may be empty.
|
|
Optional bool
|
|
// RequiredIf makes an optional field required when the rest of the
|
|
// site needs it.
|
|
requiredIf func(s Site) bool
|
|
check func(v string, s Site) error
|
|
}
|
|
|
|
// Site holds a site's values by Field.Key.
|
|
type Site map[string]string
|
|
|
|
const (
|
|
groupNames = "Names"
|
|
groupProxmox = "Proxmox"
|
|
groupNetwork = "Management network"
|
|
groupHeads = "Head nodes and images"
|
|
groupAccess = "How users reach the portal"
|
|
)
|
|
|
|
// Fields is ordered as the form presents it.
|
|
var Fields = []Field{
|
|
{Key: "PORTAL_DOMAIN", Group: groupNames, Label: "Portal domain",
|
|
Help: "Where the portal answers, e.g. hpc.example.org.", check: hostname},
|
|
{Key: "CLUSTER_DOMAIN", Group: groupNames, Label: "Cluster domain",
|
|
Help: "Each cluster's portal is c<NNN>.<this>; needs a wildcard DNS record. Usually the same as the portal domain.",
|
|
check: hostname},
|
|
|
|
{Key: "SERVE_MODE", Group: groupAccess, Label: "Serving",
|
|
Help: "proxy: behind your own reverse proxy and certificate. direct: hpcod serves HTTPS itself, from a CA of its own - for a closed network with no public domain.",
|
|
Default: "proxy", check: oneOf("proxy", "direct")},
|
|
{Key: "SERVE_DNS", Group: groupAccess, Label: "Answer DNS for these names",
|
|
Help: "true runs a DNS server for the portal and *.cluster names; point users' resolvers (or a forwarder for the zone) at PUBLIC_IP.",
|
|
Default: "false", check: boolean},
|
|
{Key: "PUBLIC_IP", Group: groupAccess, Label: "Address users reach this host at",
|
|
Help: "Where HTTPS and DNS listen, and what the names resolve to. Needed for direct serving or DNS.",
|
|
Optional: true, requiredIf: func(s Site) bool { return s.Direct() || s.DNS() }, check: ipv4},
|
|
|
|
{Key: "PVE_MODE", Group: groupProxmox, Label: "Proxmox to build clusters on",
|
|
Help: "real: a Proxmox VE you have. sim: a simulated one, in QEMU on this host, to try hpcod - setup fills in every Proxmox and network value below.",
|
|
Default: "real", check: oneOf("real", "sim")},
|
|
{Key: "SIM_CORES", Group: groupProxmox, Label: "Simulated Proxmox: cores", Default: "8", check: positive},
|
|
{Key: "SIM_MEMORY_GB", Group: groupProxmox, Label: "Simulated Proxmox: memory (GiB)", Default: "24", check: positive},
|
|
{Key: "SIM_DISK_GB", Group: groupProxmox, Label: "Simulated Proxmox: disk (GiB)", Default: "150", check: positive},
|
|
{Key: "PVE_ENDPOINT", Group: groupProxmox, Label: "Proxmox API",
|
|
Help: "e.g. https://192.0.2.10:8006", check: endpoint},
|
|
{Key: "PVE_INSECURE_TLS", Group: groupProxmox, Label: "Accept a self-signed certificate",
|
|
Help: "true while the API presents Proxmox's own certificate.", Default: "true", check: boolean},
|
|
{Key: "PVE_HOST", Group: groupProxmox, Label: "Proxmox SSH address",
|
|
Help: "Where this host reaches the hypervisor as root over SSH. Usually the API's host.", check: hostOrIP},
|
|
{Key: "PVE_NODE", Group: groupProxmox, Label: "Proxmox node",
|
|
Help: "The node the template, the snippets and every head node live on.", check: name},
|
|
{Key: "PVE_NODES", Group: groupProxmox, Label: "Nodes clusters may use",
|
|
Help: "Proxmox nodes compute nodes may be placed on, separated by commas. Empty means PVE_NODE alone. More than one needs a vxlan SDN zone.",
|
|
Optional: true, check: nodeList},
|
|
{Key: "PVE_SITE_ID", Group: groupProxmox, Label: "Site id",
|
|
Help: "What the portal calls this Proxmox; names its token variable.", Default: "hv01", check: name},
|
|
{Key: "PVE_SITE_NAME", Group: groupProxmox, Label: "Site name",
|
|
Help: "Shown on the portal's pages.", Default: "Proxmox"},
|
|
{Key: "PVE_DATASTORE", Group: groupProxmox, Label: "Datastore for VM disks", Default: "local-zfs", check: name},
|
|
{Key: "PVE_SNIPPET_DATASTORE", Group: groupProxmox, Label: "Datastore for cloud-init snippets",
|
|
Help: "Setup enables the snippets content type on it.", Default: "local", check: name},
|
|
{Key: "PVE_IMAGE", Group: groupProxmox, Label: "Cloud image",
|
|
Help: "storage:import/file. Downloaded if missing.",
|
|
Default: "local:import/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2", check: volume},
|
|
{Key: "PVE_SDN_ZONE", Group: groupProxmox, Label: "SDN zone",
|
|
Help: "Created if missing; every cluster gets a vnet in it.", Default: "hpcod", check: sdnZone},
|
|
{Key: "PVE_SDN_ZONE_TYPE", Group: groupProxmox, Label: "SDN zone type",
|
|
Help: "auto: simple on one node, vxlan across several. simple gives each node its own bridge, so it cannot carry a cluster across nodes; vxlan can.",
|
|
Default: "auto", check: zoneType},
|
|
|
|
{Key: "MGMT_BRIDGE", Group: groupNetwork, Label: "Management bridge", Default: "vmbr0", check: name},
|
|
{Key: "MGMT_GATEWAY", Group: groupNetwork, Label: "Gateway", Help: "Only /24 networks are supported.", check: ipv4},
|
|
{Key: "MGMT_FIRST", Group: groupNetwork, Label: "First head node address",
|
|
Help: "A run of free addresses outside DHCP, one per live cluster.", check: inGatewayNet},
|
|
{Key: "MGMT_COUNT", Group: groupNetwork, Label: "How many addresses", Default: "40", check: mgmtCount},
|
|
{Key: "BUILD_IP", Group: groupNetwork, Label: "Template build address",
|
|
Help: "One more free address, for the VM the head template is built in.", check: buildIP},
|
|
{Key: "NAMESERVERS", Group: groupNetwork, Label: "DNS servers", Default: "1.1.1.1 8.8.8.8", check: ipList},
|
|
|
|
{Key: "HEAD_TEMPLATE_VMID", Group: groupHeads, Label: "Head template VMID", Default: "9100", check: vmid},
|
|
{Key: "CI_USER", Group: groupHeads, Label: "Head node login account",
|
|
Help: "Created by cloud-init on every head; the portal archives /home through it.", Default: "hpcod", check: unixUser},
|
|
{Key: "EXTRA_SSH_KEYS", Group: groupHeads, Label: "Extra SSH public keys", Optional: true,
|
|
Help: "Installed for that account on every head, separated by ';'. This host's key is always included."},
|
|
{Key: "REGISTRY_HOST", Group: groupHeads, Label: "Image registry", Default: "gitea.mynodes.xyz", check: hostOrIP},
|
|
{Key: "NODE_IMAGE_URI", Group: groupHeads, Label: "Compute node image",
|
|
Default: "docker://gitea.mynodes.xyz/hpcod/node-image:rocky9-ohpc3", check: imageURI},
|
|
{Key: "DESKTOP_IMAGE", Group: groupHeads, Label: "Desktop image",
|
|
Default: "docker://gitea.mynodes.xyz/hpcapps/desktop:xfce", check: imageURI},
|
|
}
|
|
|
|
// FieldByKey finds a field's description.
|
|
func FieldByKey(k string) (Field, bool) {
|
|
for _, f := range Fields {
|
|
if f.Key == k {
|
|
return f, true
|
|
}
|
|
}
|
|
return Field{}, false
|
|
}
|
|
|
|
// Defaults is a site with every default filled in.
|
|
func Defaults() Site {
|
|
s := Site{}
|
|
for _, f := range Fields {
|
|
s[f.Key] = f.Default
|
|
}
|
|
return s
|
|
}
|
|
|
|
// Check validates one value in the context of the rest of the site.
|
|
func (f Field) Check(v string, s Site) error {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
if f.Optional && (f.requiredIf == nil || !f.requiredIf(s)) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("required")
|
|
}
|
|
if strings.ContainsAny(v, "\"$`\\\n") {
|
|
return fmt.Errorf("may not contain quotes, $, backquotes or backslashes")
|
|
}
|
|
if f.check != nil {
|
|
return f.check(v, s)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Validate checks every field, and returns every problem rather than the
|
|
// first: a form that reports one error per submission is a form that
|
|
// takes eight submissions.
|
|
func (s Site) Validate() []error {
|
|
var errs []error
|
|
for _, f := range Fields {
|
|
if err := f.Check(s[f.Key], s); err != nil {
|
|
errs = append(errs, fmt.Errorf("%s (%s): %w", f.Label, f.Key, err))
|
|
}
|
|
}
|
|
return errs
|
|
}
|
|
|
|
// TokenVar is the environment variable the discovery token lives in:
|
|
// hv01 -> HV01_TOKEN.
|
|
func (s Site) TokenVar() string {
|
|
b := []byte(strings.ToUpper(s["PVE_SITE_ID"]) + "_TOKEN")
|
|
for i, c := range b {
|
|
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
|
|
b[i] = '_'
|
|
}
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// Nodes are the Proxmox nodes clusters may use, PVE_NODE first. PVE_NODE
|
|
// is always one of them: it holds the template, so every head runs there.
|
|
func (s Site) Nodes() []string {
|
|
out := []string{s["PVE_NODE"]}
|
|
for _, n := range strings.FieldsFunc(s["PVE_NODES"], func(r rune) bool { return r == ',' || r == ' ' }) {
|
|
if !contains(out, n) {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ZoneType is the SDN zone type the site needs: what PVE_SDN_ZONE_TYPE
|
|
// says, with auto resolved by the number of nodes.
|
|
func (s Site) ZoneType() string {
|
|
if t := s["PVE_SDN_ZONE_TYPE"]; t == "simple" || t == "vxlan" {
|
|
return t
|
|
}
|
|
if len(s.Nodes()) > 1 {
|
|
return "vxlan"
|
|
}
|
|
return "simple"
|
|
}
|
|
|
|
func contains(list []string, v string) bool {
|
|
for _, x := range list {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Direct reports whether the portal serves HTTPS itself.
|
|
func (s Site) Direct() bool { return s["SERVE_MODE"] == "direct" }
|
|
|
|
// DNS reports whether the portal answers DNS for its names.
|
|
func (s Site) DNS() bool { return s["SERVE_DNS"] == "true" }
|
|
|
|
// CertNames are the names the portal's certificate covers.
|
|
func (s Site) CertNames() []string {
|
|
out := []string{s["PORTAL_DOMAIN"]}
|
|
for _, n := range []string{s["CLUSTER_DOMAIN"], "*." + s["CLUSTER_DOMAIN"]} {
|
|
if n != out[0] {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ExtraKeys splits EXTRA_SSH_KEYS.
|
|
func (s Site) ExtraKeys() []string {
|
|
var out []string
|
|
for _, k := range strings.Split(s["EXTRA_SSH_KEYS"], ";") {
|
|
if k = strings.TrimSpace(k); k != "" {
|
|
out = append(out, k)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// LoadSite reads a site.env: KEY=value lines, optional double quotes. Keys
|
|
// it does not know are kept, so a file written by a newer binary survives
|
|
// a round trip through an older one.
|
|
func LoadSite(path string) (Site, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer f.Close()
|
|
s := Site{}
|
|
sc := bufio.NewScanner(f)
|
|
for sc.Scan() {
|
|
line := strings.TrimSpace(sc.Text())
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
k, v, ok := strings.Cut(line, "=")
|
|
if !ok {
|
|
continue
|
|
}
|
|
v = strings.TrimSpace(v)
|
|
if len(v) >= 2 && v[0] == '"' && v[len(v)-1] == '"' {
|
|
v = v[1 : len(v)-1]
|
|
}
|
|
s[strings.TrimSpace(k)] = v
|
|
}
|
|
return s, sc.Err()
|
|
}
|
|
|
|
// Render writes the site in site.env form, grouped and commented, so the
|
|
// file the form saves reads like the one a person would write.
|
|
func (s Site) Render() string {
|
|
var b strings.Builder
|
|
b.WriteString("# This installation's values. Written by `hpcod setup`; safe to edit by\n")
|
|
b.WriteString("# hand and re-run it. Nothing here is secret.\n")
|
|
if h := s["DEPLOY_HOST"]; h != "" {
|
|
fmt.Fprintf(&b, "\n# Where `make deploy` installs from a workstation.\nDEPLOY_HOST=%s\n", h)
|
|
}
|
|
group := ""
|
|
for _, f := range Fields {
|
|
if f.Group != group {
|
|
group = f.Group
|
|
fmt.Fprintf(&b, "\n# --- %s\n", group)
|
|
}
|
|
if f.Help != "" {
|
|
fmt.Fprintf(&b, "# %s\n", f.Help)
|
|
}
|
|
v := s[f.Key]
|
|
if strings.ContainsAny(v, " ;#") || v == "" {
|
|
v = `"` + v + `"`
|
|
}
|
|
fmt.Fprintf(&b, "%s=%s\n", f.Key, v)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// --- checks -----------------------------------------------------------------
|
|
|
|
var (
|
|
reName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`)
|
|
reHost = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}$`)
|
|
reUnixUser = regexp.MustCompile(`^[a-z_][a-z0-9_-]{0,31}$`)
|
|
reSDNZone = regexp.MustCompile(`^[a-z][a-z0-9]{0,7}$`)
|
|
)
|
|
|
|
func name(v string, _ Site) error {
|
|
if !reName.MatchString(v) {
|
|
return fmt.Errorf("letters, digits, '.', '_' and '-' only")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func hostname(v string, _ Site) error {
|
|
if !reHost.MatchString(v) {
|
|
return fmt.Errorf("not a domain name")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func hostOrIP(v string, s Site) error {
|
|
if net.ParseIP(v) != nil {
|
|
return nil
|
|
}
|
|
return hostname(v, s)
|
|
}
|
|
|
|
func endpoint(v string, _ Site) error {
|
|
u, err := url.Parse(v)
|
|
if err != nil || u.Scheme != "https" || u.Host == "" || (u.Path != "" && u.Path != "/") {
|
|
return fmt.Errorf("expected https://host:8006")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func oneOf(vals ...string) func(string, Site) error {
|
|
return func(v string, _ Site) error {
|
|
for _, x := range vals {
|
|
if v == x {
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("one of: %s", strings.Join(vals, ", "))
|
|
}
|
|
}
|
|
|
|
func boolean(v string, _ Site) error {
|
|
if v != "true" && v != "false" {
|
|
return fmt.Errorf("true or false")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func volume(v string, _ Site) error {
|
|
store, file, ok := strings.Cut(v, ":import/")
|
|
if !ok || store == "" || file == "" || strings.Contains(file, "/") {
|
|
return fmt.Errorf("expected storage:import/file.qcow2")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func sdnZone(v string, _ Site) error {
|
|
// Proxmox's own rule for zone ids.
|
|
if !reSDNZone.MatchString(v) {
|
|
return fmt.Errorf("lowercase letter, then up to 7 lowercase letters or digits")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func nodeList(v string, s Site) error {
|
|
names := strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' })
|
|
for _, n := range names {
|
|
if err := name(n, s); err != nil {
|
|
return fmt.Errorf("%s: %w", n, err)
|
|
}
|
|
}
|
|
// Written out in full or not at all: a list that leaves out the
|
|
// template's node reads as if heads could not run there.
|
|
if len(names) > 0 && s["PVE_NODE"] != "" && !contains(names, s["PVE_NODE"]) {
|
|
return fmt.Errorf("must include %s, where the template and the heads are", s["PVE_NODE"])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func zoneType(v string, s Site) error {
|
|
if err := oneOf("auto", "simple", "vxlan")(v, s); err != nil {
|
|
return err
|
|
}
|
|
if v == "simple" && len(s.Nodes()) > 1 {
|
|
return fmt.Errorf("a simple zone is one bridge per node, so a cluster across %d nodes would be split; use vxlan or auto", len(s.Nodes()))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ipv4(v string, _ Site) error {
|
|
ip := net.ParseIP(v)
|
|
if ip == nil || ip.To4() == nil {
|
|
return fmt.Errorf("not an IPv4 address")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ipList(v string, s Site) error {
|
|
for _, ip := range strings.Fields(v) {
|
|
if err := ipv4(ip, s); err != nil {
|
|
return fmt.Errorf("%s: %w", ip, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// net24 is the /24 the gateway is on, which is the management network:
|
|
// the head node prefix is fixed at 24.
|
|
func net24(ip string) string {
|
|
p := net.ParseIP(ip).To4()
|
|
if p == nil {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf("%d.%d.%d", p[0], p[1], p[2])
|
|
}
|
|
|
|
func lastOctet(ip string) int {
|
|
p := net.ParseIP(ip).To4()
|
|
if p == nil {
|
|
return -1
|
|
}
|
|
return int(p[3])
|
|
}
|
|
|
|
func inGatewayNet(v string, s Site) error {
|
|
if err := ipv4(v, s); err != nil {
|
|
return err
|
|
}
|
|
if g := s["MGMT_GATEWAY"]; net24(g) != "" && net24(v) != net24(g) {
|
|
return fmt.Errorf("not on the gateway's network %s.0/24", net24(g))
|
|
}
|
|
if v == s["MGMT_GATEWAY"] {
|
|
return fmt.Errorf("is the gateway")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func mgmtCount(v string, s Site) error {
|
|
n, err := strconv.Atoi(v)
|
|
if err != nil || n < 1 {
|
|
return fmt.Errorf("a positive number")
|
|
}
|
|
if first := lastOctet(s["MGMT_FIRST"]); first > 0 && first+n-1 > 254 {
|
|
return fmt.Errorf("runs past .254 from %s", s["MGMT_FIRST"])
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func buildIP(v string, s Site) error {
|
|
if err := inGatewayNet(v, s); err != nil {
|
|
return err
|
|
}
|
|
first, n := lastOctet(s["MGMT_FIRST"]), 0
|
|
n, _ = strconv.Atoi(s["MGMT_COUNT"])
|
|
if o := lastOctet(v); first > 0 && net24(v) == net24(s["MGMT_FIRST"]) && o >= first && o < first+n {
|
|
return fmt.Errorf("inside the head node window; a cluster could be given it")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func positive(v string, _ Site) error {
|
|
if n, err := strconv.Atoi(v); err != nil || n < 1 {
|
|
return fmt.Errorf("a positive number")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func vmid(v string, _ Site) error {
|
|
n, err := strconv.Atoi(v)
|
|
if err != nil || n < 100 || n > 999999999 {
|
|
return fmt.Errorf("a VMID, 100 or more")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func unixUser(v string, _ Site) error {
|
|
if !reUnixUser.MatchString(v) || v == "root" {
|
|
return fmt.Errorf("a login name, not root")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func imageURI(v string, _ Site) error {
|
|
if !strings.HasPrefix(v, "docker://") && !strings.HasPrefix(v, "oras://") {
|
|
return fmt.Errorf("expected docker://… or oras://…")
|
|
}
|
|
return nil
|
|
}
|