Files
ilker f74859d138 hpcod phase 1: multi-provider discovery
First code for the HPC-on-demand portal. The portal is meant to front more
than Proxmox - OpenStack is expected next - so the provider abstraction comes
first and Proxmox is simply its first implementation. internal/model holds the
vocabulary everything above reasons in; adding a provider means writing one
package and registering it.

The layered capability detection from the design is implemented and enforced.
A PCI class code cannot identify InfiniBand: a ConnectX-4 Lx reports class
0x0200 because that variant is Ethernet-only, while a ConnectX-6 reports
0x0207 in IB mode and 0x0200 in Ethernet mode. Every capability claim
therefore records how it was made - class, curated vendor:device map, or live
agent - and Capability.Trusted refuses to schedule on a fabric until an agent
confirms the link. Unconfirmed devices are still reported, since that gap is
what an operator has to close. Passthrough viability comes from IOMMU group
membership rather than device presence: sibling functions of one card are
fine, an unrelated device in the group is not.

Verified against a live Proxmox site, where it correctly classifies the
Mellanox cards as roce rather than infiniband and withholds them from the
schedulable set, while offering the Quadro K620 that does sit alone in its
IOMMU group.

Two bugs found and fixed while running it for real:

- Query strings were appended to url.URL.Path, which is the decoded path, so
  String() escaped the "?" and /cluster/resources?type=vm silently stopped
  filtering. Guests were ignored and a node running 22 VMs reported all cores
  free. Regression test added.
- Physical cores are now the schedulable total by default rather than being
  ignored, with a per-site cpu_overcommit for hosts where oversubscription is
  acceptable. An HPC tenant given oversubscribed cores just runs slowly.

Secrets stay out of the catalogue: any setting may be written env:NAME. The
Proxmox token needs PVEAuditor only.
2026-09-01 11:53:35 +02:00

147 lines
4.6 KiB
Go

package capability
import (
"testing"
"github.com/ilkermanap/hpcod/internal/model"
)
// The Quadro K620 and ConnectX-4 Lx rows are the real values read from a live
// Proxmox node, which is what surfaced the InfiniBand problem in the first
// place.
func TestClassify(t *testing.T) {
cases := []struct {
name string
in Input
wantOK bool
wantCap model.Capability
wantConf model.Confidence
}{
{
name: "nvidia gpu by class",
in: Input{NativeID: "0000:01:00.0", Class: "0x030000", Vendor: "0x10de",
Device: "0x13bb", VendorName: "NVIDIA Corporation",
DeviceName: "GM107GL [Quadro K620]"},
wantOK: true, wantCap: model.CapGPU, wantConf: model.ConfClass,
},
{
name: "compute gpu, 3D class",
in: Input{Class: "0x030200", Vendor: "0x10de", Device: "0x1db4",
DeviceName: "GV100GL [Tesla V100]"},
wantOK: true, wantCap: model.CapGPU, wantConf: model.ConfClass,
},
{
// The case the whole design turns on: this card reports the
// Ethernet class, and only the curated map knows it can never
// do InfiniBand.
name: "connectx-4 lx is roce, not infiniband",
in: Input{NativeID: "0000:07:00.0", Class: "0x020000", Vendor: "0x15b3",
Device: "0x1015", VendorName: "Mellanox Technologies",
DeviceName: "MT27710 Family [ConnectX-4 Lx]"},
wantOK: true, wantCap: model.CapRoCE, wantConf: model.ConfMap,
},
{
// Same silicon family, IB-capable, currently in Ethernet mode.
// The map catches what the class hides.
name: "connectx-6 in ethernet mode still known IB-capable",
in: Input{Class: "0x020000", Vendor: "0x15b3", Device: "0x101b",
DeviceName: "MT2892 Family [ConnectX-6 Dx]"},
wantOK: true, wantCap: model.CapInfiniBand, wantConf: model.ConfMap,
},
{
name: "unknown ethernet nic is not a capability",
in: Input{Class: "0x020000", Vendor: "0x8086", Device: "0x153a"},
wantOK: false,
},
{
name: "nvme by class",
in: Input{Class: "0x010802", Vendor: "0x144d", Device: "0xa808"},
wantOK: true, wantCap: model.CapNVMe, wantConf: model.ConfClass,
},
{
name: "usb controller ignored",
in: Input{Class: "0x0c0330", Vendor: "0x1022", Device: "0x149c"},
wantOK: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, ok := Classify(tc.in)
if ok != tc.wantOK {
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
}
if !ok {
return
}
if got.Capability != tc.wantCap {
t.Errorf("capability = %q, want %q", got.Capability, tc.wantCap)
}
if got.Confidence != tc.wantConf {
t.Errorf("confidence = %q, want %q", got.Confidence, tc.wantConf)
}
})
}
}
// Fabric capabilities must not be schedulable until an agent confirms them,
// however they were detected. Presence-based capabilities may be.
func TestTrustRules(t *testing.T) {
cases := []struct {
cap model.Capability
conf model.Confidence
want bool
}{
{model.CapGPU, model.ConfClass, true},
{model.CapGPU, model.ConfMap, true},
{model.CapNVMe, model.ConfClass, true},
{model.CapInfiniBand, model.ConfClass, false},
{model.CapInfiniBand, model.ConfMap, false},
{model.CapInfiniBand, model.ConfAgent, true},
{model.CapRoCE, model.ConfMap, false},
{model.CapRoCE, model.ConfAgent, true},
}
for _, tc := range cases {
if got := tc.cap.Trusted(tc.conf); got != tc.want {
t.Errorf("%s/%s trusted = %v, want %v", tc.cap, tc.conf, got, tc.want)
}
}
}
// A site must not advertise a fabric it has only inferred. This is the
// end-to-end version of the rule above.
func TestSiteDoesNotAdvertiseUnconfirmedFabric(t *testing.T) {
s := model.Site{Nodes: []model.Node{{
Name: "n1", Online: true,
Devices: []model.Device{
{Capability: model.CapGPU, Confidence: model.ConfClass, Passthrough: true},
{Capability: model.CapInfiniBand, Confidence: model.ConfMap, Passthrough: true},
},
}}}
adv := s.Capabilities()
if adv[model.CapGPU] != 1 {
t.Errorf("gpu advertised = %d, want 1", adv[model.CapGPU])
}
if n, ok := adv[model.CapInfiniBand]; ok {
t.Errorf("infiniband advertised = %d from map-confidence alone; want it withheld", n)
}
if s.Claimed()[model.CapInfiniBand] != 1 {
t.Error("claimed() should still report the unconfirmed device so operators can see it")
}
}
// A device that cannot be passed through is not allocatable even if present.
func TestPassthroughGatesAdvertisement(t *testing.T) {
s := model.Site{Nodes: []model.Node{{
Name: "n1", Online: true,
Devices: []model.Device{
{Capability: model.CapGPU, Confidence: model.ConfClass, Passthrough: false},
},
}}}
if n := s.Capabilities()[model.CapGPU]; n != 0 {
t.Errorf("advertised %d GPUs that cannot be passed through", n)
}
}