First code for the HPC-on-demand portal. The portal is meant to front more than Proxmox - OpenStack is expected next - so the provider abstraction comes first and Proxmox is simply its first implementation. internal/model holds the vocabulary everything above reasons in; adding a provider means writing one package and registering it. The layered capability detection from the design is implemented and enforced. A PCI class code cannot identify InfiniBand: a ConnectX-4 Lx reports class 0x0200 because that variant is Ethernet-only, while a ConnectX-6 reports 0x0207 in IB mode and 0x0200 in Ethernet mode. Every capability claim therefore records how it was made - class, curated vendor:device map, or live agent - and Capability.Trusted refuses to schedule on a fabric until an agent confirms the link. Unconfirmed devices are still reported, since that gap is what an operator has to close. Passthrough viability comes from IOMMU group membership rather than device presence: sibling functions of one card are fine, an unrelated device in the group is not. Verified against a live Proxmox site, where it correctly classifies the Mellanox cards as roce rather than infiniband and withholds them from the schedulable set, while offering the Quadro K620 that does sit alone in its IOMMU group. Two bugs found and fixed while running it for real: - Query strings were appended to url.URL.Path, which is the decoded path, so String() escaped the "?" and /cluster/resources?type=vm silently stopped filtering. Guests were ignored and a node running 22 VMs reported all cores free. Regression test added. - Physical cores are now the schedulable total by default rather than being ignored, with a per-site cpu_overcommit for hosts where oversubscription is acceptable. An HPC tenant given oversubscribed cores just runs slowly. Secrets stay out of the catalogue: any setting may be written env:NAME. The Proxmox token needs PVEAuditor only.
147 lines
4.6 KiB
Go
147 lines
4.6 KiB
Go
package capability
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/model"
|
|
)
|
|
|
|
// The Quadro K620 and ConnectX-4 Lx rows are the real values read from a live
|
|
// Proxmox node, which is what surfaced the InfiniBand problem in the first
|
|
// place.
|
|
func TestClassify(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
in Input
|
|
wantOK bool
|
|
wantCap model.Capability
|
|
wantConf model.Confidence
|
|
}{
|
|
{
|
|
name: "nvidia gpu by class",
|
|
in: Input{NativeID: "0000:01:00.0", Class: "0x030000", Vendor: "0x10de",
|
|
Device: "0x13bb", VendorName: "NVIDIA Corporation",
|
|
DeviceName: "GM107GL [Quadro K620]"},
|
|
wantOK: true, wantCap: model.CapGPU, wantConf: model.ConfClass,
|
|
},
|
|
{
|
|
name: "compute gpu, 3D class",
|
|
in: Input{Class: "0x030200", Vendor: "0x10de", Device: "0x1db4",
|
|
DeviceName: "GV100GL [Tesla V100]"},
|
|
wantOK: true, wantCap: model.CapGPU, wantConf: model.ConfClass,
|
|
},
|
|
{
|
|
// The case the whole design turns on: this card reports the
|
|
// Ethernet class, and only the curated map knows it can never
|
|
// do InfiniBand.
|
|
name: "connectx-4 lx is roce, not infiniband",
|
|
in: Input{NativeID: "0000:07:00.0", Class: "0x020000", Vendor: "0x15b3",
|
|
Device: "0x1015", VendorName: "Mellanox Technologies",
|
|
DeviceName: "MT27710 Family [ConnectX-4 Lx]"},
|
|
wantOK: true, wantCap: model.CapRoCE, wantConf: model.ConfMap,
|
|
},
|
|
{
|
|
// Same silicon family, IB-capable, currently in Ethernet mode.
|
|
// The map catches what the class hides.
|
|
name: "connectx-6 in ethernet mode still known IB-capable",
|
|
in: Input{Class: "0x020000", Vendor: "0x15b3", Device: "0x101b",
|
|
DeviceName: "MT2892 Family [ConnectX-6 Dx]"},
|
|
wantOK: true, wantCap: model.CapInfiniBand, wantConf: model.ConfMap,
|
|
},
|
|
{
|
|
name: "unknown ethernet nic is not a capability",
|
|
in: Input{Class: "0x020000", Vendor: "0x8086", Device: "0x153a"},
|
|
wantOK: false,
|
|
},
|
|
{
|
|
name: "nvme by class",
|
|
in: Input{Class: "0x010802", Vendor: "0x144d", Device: "0xa808"},
|
|
wantOK: true, wantCap: model.CapNVMe, wantConf: model.ConfClass,
|
|
},
|
|
{
|
|
name: "usb controller ignored",
|
|
in: Input{Class: "0x0c0330", Vendor: "0x1022", Device: "0x149c"},
|
|
wantOK: false,
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, ok := Classify(tc.in)
|
|
if ok != tc.wantOK {
|
|
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
if got.Capability != tc.wantCap {
|
|
t.Errorf("capability = %q, want %q", got.Capability, tc.wantCap)
|
|
}
|
|
if got.Confidence != tc.wantConf {
|
|
t.Errorf("confidence = %q, want %q", got.Confidence, tc.wantConf)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Fabric capabilities must not be schedulable until an agent confirms them,
|
|
// however they were detected. Presence-based capabilities may be.
|
|
func TestTrustRules(t *testing.T) {
|
|
cases := []struct {
|
|
cap model.Capability
|
|
conf model.Confidence
|
|
want bool
|
|
}{
|
|
{model.CapGPU, model.ConfClass, true},
|
|
{model.CapGPU, model.ConfMap, true},
|
|
{model.CapNVMe, model.ConfClass, true},
|
|
|
|
{model.CapInfiniBand, model.ConfClass, false},
|
|
{model.CapInfiniBand, model.ConfMap, false},
|
|
{model.CapInfiniBand, model.ConfAgent, true},
|
|
{model.CapRoCE, model.ConfMap, false},
|
|
{model.CapRoCE, model.ConfAgent, true},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := tc.cap.Trusted(tc.conf); got != tc.want {
|
|
t.Errorf("%s/%s trusted = %v, want %v", tc.cap, tc.conf, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A site must not advertise a fabric it has only inferred. This is the
|
|
// end-to-end version of the rule above.
|
|
func TestSiteDoesNotAdvertiseUnconfirmedFabric(t *testing.T) {
|
|
s := model.Site{Nodes: []model.Node{{
|
|
Name: "n1", Online: true,
|
|
Devices: []model.Device{
|
|
{Capability: model.CapGPU, Confidence: model.ConfClass, Passthrough: true},
|
|
{Capability: model.CapInfiniBand, Confidence: model.ConfMap, Passthrough: true},
|
|
},
|
|
}}}
|
|
|
|
adv := s.Capabilities()
|
|
if adv[model.CapGPU] != 1 {
|
|
t.Errorf("gpu advertised = %d, want 1", adv[model.CapGPU])
|
|
}
|
|
if n, ok := adv[model.CapInfiniBand]; ok {
|
|
t.Errorf("infiniband advertised = %d from map-confidence alone; want it withheld", n)
|
|
}
|
|
if s.Claimed()[model.CapInfiniBand] != 1 {
|
|
t.Error("claimed() should still report the unconfirmed device so operators can see it")
|
|
}
|
|
}
|
|
|
|
// A device that cannot be passed through is not allocatable even if present.
|
|
func TestPassthroughGatesAdvertisement(t *testing.T) {
|
|
s := model.Site{Nodes: []model.Node{{
|
|
Name: "n1", Online: true,
|
|
Devices: []model.Device{
|
|
{Capability: model.CapGPU, Confidence: model.ConfClass, Passthrough: false},
|
|
},
|
|
}}}
|
|
if n := s.Capabilities()[model.CapGPU]; n != 0 {
|
|
t.Errorf("advertised %d GPUs that cannot be passed through", n)
|
|
}
|
|
}
|