ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
237 lines
6.3 KiB
Go
237 lines
6.3 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// vxlanMTU is a 1500-byte network less the 50 bytes VXLAN wraps every
|
|
// frame in. Guests are given it through virtio (mtu=1 in the module).
|
|
const vxlanMTU = "1450"
|
|
|
|
// zoneState is what Proxmox says about a zone, as far as setup cares.
|
|
type zoneState struct {
|
|
Type string
|
|
Peers []string
|
|
Nodes []string
|
|
MTU string
|
|
}
|
|
|
|
// readZone returns the zone, or nil when there is none.
|
|
func readZone(ctx context.Context, p *PVE, zone string) (*zoneState, error) {
|
|
var all []map[string]any
|
|
if err := p.Get(ctx, "/cluster/sdn/zones", &all); err != nil {
|
|
return nil, fmt.Errorf("listing SDN zones: %w", err)
|
|
}
|
|
for _, z := range all {
|
|
if fmt.Sprint(z["zone"]) != zone {
|
|
continue
|
|
}
|
|
st := &zoneState{Type: fmt.Sprint(z["type"])}
|
|
st.Peers = splitComma(z["peers"])
|
|
st.Nodes = splitComma(z["nodes"])
|
|
if v, ok := z["mtu"]; ok && v != nil {
|
|
st.MTU = fmt.Sprint(v)
|
|
}
|
|
return st, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func splitComma(v any) []string {
|
|
if v == nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for _, x := range strings.Split(fmt.Sprint(v), ",") {
|
|
if x = strings.TrimSpace(x); x != "" {
|
|
out = append(out, x)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// nodeAddresses checks that every node the site uses exists and is online,
|
|
// and returns each one's cluster address - what VXLAN peers are.
|
|
func nodeAddresses(ctx context.Context, p *PVE, nodes []string) (map[string]string, error) {
|
|
var ns []struct {
|
|
Node string `json:"node"`
|
|
Status string `json:"status"`
|
|
}
|
|
if err := p.Get(ctx, "/nodes", &ns); err != nil {
|
|
return nil, err
|
|
}
|
|
status := map[string]string{}
|
|
for _, n := range ns {
|
|
status[n.Node] = n.Status
|
|
}
|
|
var st []struct {
|
|
Type string `json:"type"`
|
|
Name string `json:"name"`
|
|
IP string `json:"ip"`
|
|
}
|
|
if err := p.Get(ctx, "/cluster/status", &st); err != nil {
|
|
return nil, err
|
|
}
|
|
ips := map[string]string{}
|
|
for _, x := range st {
|
|
if x.Type == "node" {
|
|
ips[x.Name] = x.IP
|
|
}
|
|
}
|
|
out := map[string]string{}
|
|
for _, n := range nodes {
|
|
switch {
|
|
case status[n] == "":
|
|
return nil, fmt.Errorf("no node %q in this Proxmox", n)
|
|
case status[n] != "online":
|
|
return nil, fmt.Errorf("node %s is %s", n, status[n])
|
|
}
|
|
out[n] = ips[n]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// zoneVnets lists the vnets in a zone: each is a live cluster's fabric.
|
|
func zoneVnets(ctx context.Context, p *PVE, zone string) ([]string, error) {
|
|
var vs []struct {
|
|
Vnet string `json:"vnet"`
|
|
Zone string `json:"zone"`
|
|
}
|
|
if err := p.Get(ctx, "/cluster/sdn/vnets", &vs); err != nil {
|
|
return nil, err
|
|
}
|
|
var out []string
|
|
for _, v := range vs {
|
|
if v.Zone == zone {
|
|
out = append(out, v.Vnet)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// wantZone is the zone a site needs, with VXLAN peers resolved.
|
|
func wantZone(s Site, ips map[string]string) (zoneState, error) {
|
|
w := zoneState{Type: s.ZoneType()}
|
|
if w.Type != "vxlan" {
|
|
return w, nil
|
|
}
|
|
w.Nodes = append([]string(nil), s.Nodes()...)
|
|
sort.Strings(w.Nodes)
|
|
for _, n := range w.Nodes {
|
|
if ips[n] == "" {
|
|
return w, fmt.Errorf("Proxmox reports no cluster address for node %s, and a vxlan zone needs one for each", n)
|
|
}
|
|
w.Peers = append(w.Peers, ips[n])
|
|
}
|
|
sort.Strings(w.Peers)
|
|
w.MTU = vxlanMTU
|
|
return w, nil
|
|
}
|
|
|
|
func (z zoneState) matches(w zoneState) bool {
|
|
if z.Type != w.Type {
|
|
return false
|
|
}
|
|
if w.Type != "vxlan" {
|
|
return true
|
|
}
|
|
return strings.Join(z.Peers, ",") == strings.Join(w.Peers, ",") &&
|
|
strings.Join(z.Nodes, ",") == strings.Join(w.Nodes, ",") && z.MTU == w.MTU
|
|
}
|
|
|
|
func (w zoneState) form() url.Values {
|
|
f := url.Values{}
|
|
if w.Type == "vxlan" {
|
|
f.Set("peers", strings.Join(w.Peers, ","))
|
|
f.Set("nodes", strings.Join(w.Nodes, ","))
|
|
f.Set("mtu", w.MTU)
|
|
}
|
|
return f
|
|
}
|
|
|
|
// ensureZone makes the site's SDN zone the type its nodes need: simple on
|
|
// one node, vxlan across several, with every node a peer.
|
|
//
|
|
// A zone's type cannot be changed in place, so a wrong one is deleted and
|
|
// made again - but only while no cluster has a vnet in it. Recreating it
|
|
// under a live cluster would cut that cluster's nodes off from their head.
|
|
func ensureZone(ctx context.Context, p *PVE, s Site, logf func(string, ...any)) error {
|
|
zone := s["PVE_SDN_ZONE"]
|
|
ips, err := nodeAddresses(ctx, p, s.Nodes())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
want, err := wantZone(s, ips)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
have, err := readZone(ctx, p, zone)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if have != nil && have.matches(want) {
|
|
logf("SDN zone %s exists (%s%s)", zone, have.Type, peersNote(*have))
|
|
return nil
|
|
}
|
|
|
|
switch {
|
|
case have == nil:
|
|
f := want.form()
|
|
f.Set("zone", zone)
|
|
f.Set("type", want.Type)
|
|
f.Set("ipam", "pve")
|
|
if err := p.Post(ctx, "/cluster/sdn/zones", f, nil); err != nil {
|
|
return fmt.Errorf("creating SDN zone %s: %w", zone, err)
|
|
}
|
|
case have.Type == want.Type:
|
|
// Same type, other nodes: peers and nodes can change in place.
|
|
if err := p.Put(ctx, "/cluster/sdn/zones/"+zone, want.form()); err != nil {
|
|
return fmt.Errorf("updating SDN zone %s: %w", zone, err)
|
|
}
|
|
default:
|
|
vnets, err := zoneVnets(ctx, p, zone)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(vnets) > 0 {
|
|
return fmt.Errorf("SDN zone %s is %s and must be %s, but %d cluster(s) still have a vnet in it (%s); release them, then run setup again",
|
|
zone, have.Type, want.Type, len(vnets), strings.Join(vnets, ", "))
|
|
}
|
|
if err := p.Delete(ctx, "/cluster/sdn/zones/"+zone); err != nil {
|
|
return fmt.Errorf("removing SDN zone %s to change its type: %w", zone, err)
|
|
}
|
|
f := want.form()
|
|
f.Set("zone", zone)
|
|
f.Set("type", want.Type)
|
|
f.Set("ipam", "pve")
|
|
if err := p.Post(ctx, "/cluster/sdn/zones", f, nil); err != nil {
|
|
return fmt.Errorf("creating SDN zone %s as %s: %w", zone, want.Type, err)
|
|
}
|
|
logf("SDN zone %s was %s; made it again as %s", zone, have.Type, want.Type)
|
|
}
|
|
if err := p.Put(ctx, "/cluster/sdn", nil); err != nil {
|
|
return fmt.Errorf("applying SDN: %w", err)
|
|
}
|
|
got, err := readZone(ctx, p, zone)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if got == nil || !got.matches(want) {
|
|
return fmt.Errorf("SDN zone %s did not take: want %s%s, Proxmox has %v", zone, want.Type, peersNote(want), got)
|
|
}
|
|
logf("SDN zone %s is %s%s, applied", zone, got.Type, peersNote(*got))
|
|
return nil
|
|
}
|
|
|
|
func peersNote(z zoneState) string {
|
|
if z.Type != "vxlan" {
|
|
return ""
|
|
}
|
|
return fmt.Sprintf(", peers %s, MTU %s", strings.Join(z.Peers, " "), z.MTU)
|
|
}
|