Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

237 lines
6.3 KiB
Go

package setup
import (
"context"
"fmt"
"net/url"
"sort"
"strings"
)
// vxlanMTU is a 1500-byte network less the 50 bytes VXLAN wraps every
// frame in. Guests are given it through virtio (mtu=1 in the module).
const vxlanMTU = "1450"
// zoneState is what Proxmox says about a zone, as far as setup cares.
type zoneState struct {
Type string
Peers []string
Nodes []string
MTU string
}
// readZone returns the zone, or nil when there is none.
func readZone(ctx context.Context, p *PVE, zone string) (*zoneState, error) {
var all []map[string]any
if err := p.Get(ctx, "/cluster/sdn/zones", &all); err != nil {
return nil, fmt.Errorf("listing SDN zones: %w", err)
}
for _, z := range all {
if fmt.Sprint(z["zone"]) != zone {
continue
}
st := &zoneState{Type: fmt.Sprint(z["type"])}
st.Peers = splitComma(z["peers"])
st.Nodes = splitComma(z["nodes"])
if v, ok := z["mtu"]; ok && v != nil {
st.MTU = fmt.Sprint(v)
}
return st, nil
}
return nil, nil
}
func splitComma(v any) []string {
if v == nil {
return nil
}
var out []string
for _, x := range strings.Split(fmt.Sprint(v), ",") {
if x = strings.TrimSpace(x); x != "" {
out = append(out, x)
}
}
sort.Strings(out)
return out
}
// nodeAddresses checks that every node the site uses exists and is online,
// and returns each one's cluster address - what VXLAN peers are.
func nodeAddresses(ctx context.Context, p *PVE, nodes []string) (map[string]string, error) {
var ns []struct {
Node string `json:"node"`
Status string `json:"status"`
}
if err := p.Get(ctx, "/nodes", &ns); err != nil {
return nil, err
}
status := map[string]string{}
for _, n := range ns {
status[n.Node] = n.Status
}
var st []struct {
Type string `json:"type"`
Name string `json:"name"`
IP string `json:"ip"`
}
if err := p.Get(ctx, "/cluster/status", &st); err != nil {
return nil, err
}
ips := map[string]string{}
for _, x := range st {
if x.Type == "node" {
ips[x.Name] = x.IP
}
}
out := map[string]string{}
for _, n := range nodes {
switch {
case status[n] == "":
return nil, fmt.Errorf("no node %q in this Proxmox", n)
case status[n] != "online":
return nil, fmt.Errorf("node %s is %s", n, status[n])
}
out[n] = ips[n]
}
return out, nil
}
// zoneVnets lists the vnets in a zone: each is a live cluster's fabric.
func zoneVnets(ctx context.Context, p *PVE, zone string) ([]string, error) {
var vs []struct {
Vnet string `json:"vnet"`
Zone string `json:"zone"`
}
if err := p.Get(ctx, "/cluster/sdn/vnets", &vs); err != nil {
return nil, err
}
var out []string
for _, v := range vs {
if v.Zone == zone {
out = append(out, v.Vnet)
}
}
return out, nil
}
// wantZone is the zone a site needs, with VXLAN peers resolved.
func wantZone(s Site, ips map[string]string) (zoneState, error) {
w := zoneState{Type: s.ZoneType()}
if w.Type != "vxlan" {
return w, nil
}
w.Nodes = append([]string(nil), s.Nodes()...)
sort.Strings(w.Nodes)
for _, n := range w.Nodes {
if ips[n] == "" {
return w, fmt.Errorf("Proxmox reports no cluster address for node %s, and a vxlan zone needs one for each", n)
}
w.Peers = append(w.Peers, ips[n])
}
sort.Strings(w.Peers)
w.MTU = vxlanMTU
return w, nil
}
func (z zoneState) matches(w zoneState) bool {
if z.Type != w.Type {
return false
}
if w.Type != "vxlan" {
return true
}
return strings.Join(z.Peers, ",") == strings.Join(w.Peers, ",") &&
strings.Join(z.Nodes, ",") == strings.Join(w.Nodes, ",") && z.MTU == w.MTU
}
func (w zoneState) form() url.Values {
f := url.Values{}
if w.Type == "vxlan" {
f.Set("peers", strings.Join(w.Peers, ","))
f.Set("nodes", strings.Join(w.Nodes, ","))
f.Set("mtu", w.MTU)
}
return f
}
// ensureZone makes the site's SDN zone the type its nodes need: simple on
// one node, vxlan across several, with every node a peer.
//
// A zone's type cannot be changed in place, so a wrong one is deleted and
// made again - but only while no cluster has a vnet in it. Recreating it
// under a live cluster would cut that cluster's nodes off from their head.
func ensureZone(ctx context.Context, p *PVE, s Site, logf func(string, ...any)) error {
zone := s["PVE_SDN_ZONE"]
ips, err := nodeAddresses(ctx, p, s.Nodes())
if err != nil {
return err
}
want, err := wantZone(s, ips)
if err != nil {
return err
}
have, err := readZone(ctx, p, zone)
if err != nil {
return err
}
if have != nil && have.matches(want) {
logf("SDN zone %s exists (%s%s)", zone, have.Type, peersNote(*have))
return nil
}
switch {
case have == nil:
f := want.form()
f.Set("zone", zone)
f.Set("type", want.Type)
f.Set("ipam", "pve")
if err := p.Post(ctx, "/cluster/sdn/zones", f, nil); err != nil {
return fmt.Errorf("creating SDN zone %s: %w", zone, err)
}
case have.Type == want.Type:
// Same type, other nodes: peers and nodes can change in place.
if err := p.Put(ctx, "/cluster/sdn/zones/"+zone, want.form()); err != nil {
return fmt.Errorf("updating SDN zone %s: %w", zone, err)
}
default:
vnets, err := zoneVnets(ctx, p, zone)
if err != nil {
return err
}
if len(vnets) > 0 {
return fmt.Errorf("SDN zone %s is %s and must be %s, but %d cluster(s) still have a vnet in it (%s); release them, then run setup again",
zone, have.Type, want.Type, len(vnets), strings.Join(vnets, ", "))
}
if err := p.Delete(ctx, "/cluster/sdn/zones/"+zone); err != nil {
return fmt.Errorf("removing SDN zone %s to change its type: %w", zone, err)
}
f := want.form()
f.Set("zone", zone)
f.Set("type", want.Type)
f.Set("ipam", "pve")
if err := p.Post(ctx, "/cluster/sdn/zones", f, nil); err != nil {
return fmt.Errorf("creating SDN zone %s as %s: %w", zone, want.Type, err)
}
logf("SDN zone %s was %s; made it again as %s", zone, have.Type, want.Type)
}
if err := p.Put(ctx, "/cluster/sdn", nil); err != nil {
return fmt.Errorf("applying SDN: %w", err)
}
got, err := readZone(ctx, p, zone)
if err != nil {
return err
}
if got == nil || !got.matches(want) {
return fmt.Errorf("SDN zone %s did not take: want %s%s, Proxmox has %v", zone, want.Type, peersNote(want), got)
}
logf("SDN zone %s is %s%s, applied", zone, got.Type, peersNote(*got))
return nil
}
func peersNote(z zoneState) string {
if z.Type != "vxlan" {
return ""
}
return fmt.Sprintf(", peers %s, MTU %s", strings.Join(z.Peers, " "), z.MTU)
}