ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
241 lines
7.9 KiB
Go
241 lines
7.9 KiB
Go
// Package model holds the provider-neutral inventory types.
|
|
//
|
|
// Everything the control plane reasons about — placement, quotas, tenancy —
|
|
// is expressed here. A provider's job is to translate its own vocabulary into
|
|
// these types; nothing above this package should know whether a site is
|
|
// Proxmox, OpenStack, or something not written yet.
|
|
package model
|
|
|
|
import "time"
|
|
|
|
// Capability is the scheduling vocabulary. A request asks for these; a site
|
|
// advertises these. Provider-native details (PCI class, flavor extra_specs,
|
|
// host aggregates) are mapped into this set and then forgotten.
|
|
type Capability string
|
|
|
|
const (
|
|
CapGPU Capability = "gpu"
|
|
CapInfiniBand Capability = "infiniband"
|
|
CapRoCE Capability = "roce" // RDMA over Ethernet - NOT InfiniBand
|
|
CapNVMe Capability = "nvme"
|
|
)
|
|
|
|
// Confidence records how a capability was determined.
|
|
//
|
|
// This exists because of a concrete finding: PCI class alone cannot identify
|
|
// InfiniBand. A ConnectX-4 Lx reports class 0x0200 (Ethernet) because that
|
|
// variant is Ethernet-only, while a ConnectX-6 reports 0x0207 in IB mode and
|
|
// 0x0200 in Ethernet mode. Class-based detection therefore yields both false
|
|
// negatives and false positives, so the control plane must know how much to
|
|
// trust each claim rather than treating all of them alike.
|
|
type Confidence string
|
|
|
|
const (
|
|
// ConfClass: derived from a device class code. Reliable for GPU and NVMe.
|
|
ConfClass Confidence = "class"
|
|
// ConfMap: matched against the curated vendor:device capability map.
|
|
ConfMap Confidence = "map"
|
|
// ConfAgent: confirmed by an on-node agent that inspected live state
|
|
// (link layer, speed, driver binding). The only trustworthy source for
|
|
// fabric capabilities.
|
|
ConfAgent Confidence = "agent"
|
|
)
|
|
|
|
// Trusted reports whether a capability claim is good enough to schedule on.
|
|
// Fabric capabilities need live confirmation; presence-based ones do not.
|
|
func (c Capability) Trusted(conf Confidence) bool {
|
|
switch c {
|
|
case CapInfiniBand, CapRoCE:
|
|
return conf == ConfAgent
|
|
default:
|
|
return conf == ConfClass || conf == ConfMap || conf == ConfAgent
|
|
}
|
|
}
|
|
|
|
// Device is one allocatable piece of hardware.
|
|
type Device struct {
|
|
// NativeID is meaningful only to the provider: a PCI address on Proxmox,
|
|
// a pci_alias or resource class elsewhere.
|
|
NativeID string `json:"native_id"`
|
|
Vendor string `json:"vendor"`
|
|
Model string `json:"model"`
|
|
Capability Capability `json:"capability"`
|
|
Confidence Confidence `json:"confidence"`
|
|
// Passthrough reports whether this device can actually be handed to a
|
|
// guest. On Proxmox that means its IOMMU group holds nothing else that
|
|
// matters; a device that merely exists is not necessarily offerable.
|
|
Passthrough bool `json:"passthrough"`
|
|
// IsolationGroup is the provider's unit of passthrough granularity
|
|
// (IOMMU group on Proxmox). Empty when the provider has no such concept.
|
|
IsolationGroup string `json:"isolation_group,omitempty"`
|
|
AllocatedTo string `json:"allocated_to,omitempty"`
|
|
}
|
|
|
|
// Storage is a pool a tenant's disks can be created from.
|
|
type Storage struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
TotalBytes int64 `json:"total_bytes"`
|
|
FreeBytes int64 `json:"free_bytes"`
|
|
Shared bool `json:"shared"`
|
|
}
|
|
|
|
// Node is one hypervisor.
|
|
//
|
|
// Cores and MemBytes are what may be scheduled, which is the hardware
|
|
// multiplied by the site's oversubscription ratios. PhysicalCores and
|
|
// PhysicalMemBytes are the hardware itself. Both are reported because the
|
|
// first alone cannot be read: 168 schedulable cores is either a large
|
|
// machine or a small one with a generous policy, and those want very
|
|
// different responses from whoever is looking.
|
|
type Node struct {
|
|
Name string `json:"name"`
|
|
Online bool `json:"online"`
|
|
CPUModel string `json:"cpu_model,omitempty"`
|
|
Cores int `json:"cores"`
|
|
MemBytes int64 `json:"mem_bytes"`
|
|
PhysicalCores int `json:"physical_cores,omitempty"`
|
|
PhysicalMemBytes int64 `json:"physical_mem_bytes,omitempty"`
|
|
AllocCores int `json:"alloc_cores"`
|
|
AllocMemBytes int64 `json:"alloc_mem_bytes"`
|
|
Devices []Device `json:"devices,omitempty"`
|
|
Storages []Storage `json:"storages,omitempty"`
|
|
}
|
|
|
|
// Oversubscribed reports whether this node's schedulable figures exceed its
|
|
// hardware.
|
|
func (n Node) Oversubscribed() bool {
|
|
if n.PhysicalCores == 0 {
|
|
return false
|
|
}
|
|
return n.Cores > n.PhysicalCores || n.MemBytes > n.PhysicalMemBytes
|
|
}
|
|
|
|
func (n Node) FreeCores() int {
|
|
if f := n.Cores - n.AllocCores; f > 0 {
|
|
return f
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (n Node) FreeMemBytes() int64 {
|
|
if f := n.MemBytes - n.AllocMemBytes; f > 0 {
|
|
return f
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// Site is one backing cloud: a Proxmox cluster, an OpenStack region, ...
|
|
type Site struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
// Kind names the provider implementation ("proxmox", "openstack").
|
|
Kind string `json:"kind"`
|
|
|
|
Reachable bool `json:"reachable"`
|
|
Healthy bool `json:"healthy"` // quorate / operational
|
|
LastSeen time.Time `json:"last_seen"`
|
|
Error string `json:"error,omitempty"`
|
|
|
|
Nodes []Node `json:"nodes,omitempty"`
|
|
|
|
// HeadNode is the only node a head may be placed on, when set: the one
|
|
// holding the head template, which Proxmox cannot clone onto another
|
|
// node without shared storage. Compute nodes are diskless and go
|
|
// anywhere in Nodes.
|
|
HeadNode string `json:"head_node,omitempty"`
|
|
}
|
|
|
|
// Capabilities summarises what this site can currently schedule, counting only
|
|
// claims trustworthy enough to act on.
|
|
func (s Site) Capabilities() map[Capability]int {
|
|
out := map[Capability]int{}
|
|
for _, n := range s.Nodes {
|
|
if !n.Online {
|
|
continue
|
|
}
|
|
for _, d := range n.Devices {
|
|
if d.AllocatedTo == "" && d.Passthrough && d.Capability.Trusted(d.Confidence) {
|
|
out[d.Capability]++
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Claimed summarises every capability seen, trustworthy or not. The difference
|
|
// between this and Capabilities() is what an operator needs to fix.
|
|
func (s Site) Claimed() map[Capability]int {
|
|
out := map[Capability]int{}
|
|
for _, n := range s.Nodes {
|
|
for _, d := range n.Devices {
|
|
out[d.Capability]++
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s Site) TotalFreeCores() int {
|
|
t := 0
|
|
for _, n := range s.Nodes {
|
|
if n.Online {
|
|
t += n.FreeCores()
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
func (s Site) TotalFreeMemBytes() int64 {
|
|
var t int64
|
|
for _, n := range s.Nodes {
|
|
if n.Online {
|
|
t += n.FreeMemBytes()
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
// Reservation is a device this control plane has promised to a cluster.
|
|
//
|
|
// It exists because the hypervisor cannot answer the question on its own.
|
|
// Proxmox knows which guest holds a card, but only after that guest has been
|
|
// created, and a cluster takes minutes to provision. During that window the
|
|
// card is spoken for and still looks free, so a second request planned in
|
|
// the meantime would be promised the same one.
|
|
type Reservation struct {
|
|
Site string `json:"site"`
|
|
HostNode string `json:"host_node"`
|
|
NativeID string `json:"native_id"`
|
|
HeldBy string `json:"held_by"` // cluster accounting name
|
|
}
|
|
|
|
// ApplyReservations marks reserved devices as taken, so placement sees the
|
|
// promises alongside the attachments.
|
|
//
|
|
// A device already reported as allocated by the site keeps that label: what
|
|
// the hypervisor says is attached is more concrete than what we intend, and
|
|
// overwriting it would hide the stronger fact.
|
|
func ApplyReservations(sites []Site, res []Reservation) {
|
|
if len(res) == 0 {
|
|
return
|
|
}
|
|
held := make(map[string]string, len(res))
|
|
for _, r := range res {
|
|
held[r.Site+"/"+r.HostNode+"/"+r.NativeID] = r.HeldBy
|
|
}
|
|
for si := range sites {
|
|
for ni := range sites[si].Nodes {
|
|
devs := sites[si].Nodes[ni].Devices
|
|
for di := range devs {
|
|
if devs[di].AllocatedTo != "" {
|
|
continue
|
|
}
|
|
key := sites[si].ID + "/" + sites[si].Nodes[ni].Name + "/" + devs[di].NativeID
|
|
if by, ok := held[key]; ok {
|
|
devs[di].AllocatedTo = "reserved for " + by
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|