First code for the HPC-on-demand portal. The portal is meant to front more than Proxmox - OpenStack is expected next - so the provider abstraction comes first and Proxmox is simply its first implementation. internal/model holds the vocabulary everything above reasons in; adding a provider means writing one package and registering it. The layered capability detection from the design is implemented and enforced. A PCI class code cannot identify InfiniBand: a ConnectX-4 Lx reports class 0x0200 because that variant is Ethernet-only, while a ConnectX-6 reports 0x0207 in IB mode and 0x0200 in Ethernet mode. Every capability claim therefore records how it was made - class, curated vendor:device map, or live agent - and Capability.Trusted refuses to schedule on a fabric until an agent confirms the link. Unconfirmed devices are still reported, since that gap is what an operator has to close. Passthrough viability comes from IOMMU group membership rather than device presence: sibling functions of one card are fine, an unrelated device in the group is not. Verified against a live Proxmox site, where it correctly classifies the Mellanox cards as roce rather than infiniband and withholds them from the schedulable set, while offering the Quadro K620 that does sit alone in its IOMMU group. Two bugs found and fixed while running it for real: - Query strings were appended to url.URL.Path, which is the decoded path, so String() escaped the "?" and /cluster/resources?type=vm silently stopped filtering. Guests were ignored and a node running 22 VMs reported all cores free. Regression test added. - Physical cores are now the schedulable total by default rather than being ignored, with a per-site cpu_overcommit for hosts where oversubscription is acceptable. An HPC tenant given oversubscribed cores just runs slowly. Secrets stay out of the catalogue: any setting may be written env:NAME. The Proxmox token needs PVEAuditor only.
175 lines
5.9 KiB
Go
175 lines
5.9 KiB
Go
// Package capability turns PCI identity into the scheduling vocabulary.
|
|
//
|
|
// It is used by any provider whose inventory is expressed in PCI terms; that
|
|
// happens to be Proxmox today and would equally be an OpenStack deployment
|
|
// exposing pci_passthrough devices.
|
|
//
|
|
// The layering here is the design's central point. Detection has three tiers:
|
|
//
|
|
// class - the PCI class code. Dependable for GPUs and NVMe.
|
|
// map - a curated vendor:device table. The only way to tell a
|
|
// ConnectX-4 Lx (Ethernet-only) from a ConnectX-6 (IB-capable).
|
|
// agent - live confirmation from the node itself.
|
|
//
|
|
// Class alone cannot identify InfiniBand: an IB-capable card in Ethernet mode
|
|
// reports the Ethernet class, and an Ethernet-only card from the same family
|
|
// reports it too. Anything claiming a fabric capability from class or map data
|
|
// is therefore a hypothesis, and model.Capability.Trusted refuses to schedule
|
|
// on it until an agent has confirmed it.
|
|
package capability
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/model"
|
|
)
|
|
|
|
// PCI classes we care about. Proxmox reports these as 0xCCSSPP
|
|
// (class, subclass, prog-if), so match on the leading four hex digits.
|
|
const (
|
|
classVGA = "0x0300" // display controller, VGA compatible
|
|
class3D = "0x0302" // display controller, 3D - compute cards
|
|
classEthernet = "0x0200"
|
|
classInfiniBand = "0x0207"
|
|
classNVMe = "0x0108"
|
|
)
|
|
|
|
// Entry is one row of the curated device table.
|
|
type Entry struct {
|
|
Vendor string // lowercase hex, no 0x
|
|
Device string // lowercase hex, no 0x
|
|
Capability model.Capability
|
|
Note string
|
|
}
|
|
|
|
// table is the curated vendor:device map.
|
|
//
|
|
// It exists to resolve exactly the cases class codes get wrong. Keep it small
|
|
// and justified: every row should record something a class code cannot say.
|
|
var table = []Entry{
|
|
// Mellanox ConnectX-4 Lx. Ethernet-only silicon, so it reports the
|
|
// Ethernet class and will never do InfiniBand however it is configured.
|
|
// Without this row the card is indistinguishable from an IB-capable part
|
|
// that merely happens to be in Ethernet mode.
|
|
{"15b3", "1015", model.CapRoCE, "ConnectX-4 Lx - Ethernet/RoCE only, never IB"},
|
|
{"15b3", "1016", model.CapRoCE, "ConnectX-4 Lx virtual function"},
|
|
|
|
// IB-capable families. These report the InfiniBand class only while in IB
|
|
// mode; in Ethernet mode they look like plain NICs. The map records that
|
|
// the hardware *can* do IB - an agent still has to confirm it currently is.
|
|
{"15b3", "1017", model.CapInfiniBand, "ConnectX-5 - mode-dependent"},
|
|
{"15b3", "1019", model.CapInfiniBand, "ConnectX-5 Ex - mode-dependent"},
|
|
{"15b3", "101b", model.CapInfiniBand, "ConnectX-6 - mode-dependent"},
|
|
{"15b3", "101d", model.CapInfiniBand, "ConnectX-6 Dx - mode-dependent"},
|
|
{"15b3", "101f", model.CapInfiniBand, "ConnectX-6 Lx - Ethernet/RoCE only"},
|
|
{"15b3", "1021", model.CapInfiniBand, "ConnectX-7 - mode-dependent"},
|
|
}
|
|
|
|
// Known vendor ids, for reporting rather than logic.
|
|
var vendors = map[string]string{
|
|
"10de": "nvidia",
|
|
"1002": "amd",
|
|
"8086": "intel",
|
|
"15b3": "mellanox",
|
|
}
|
|
|
|
// Input is the provider-agnostic description of one PCI function.
|
|
type Input struct {
|
|
NativeID string // e.g. "0000:01:00.0"
|
|
Class string // e.g. "0x030000"
|
|
Vendor string // e.g. "0x10de" or "10de"
|
|
Device string
|
|
VendorName string
|
|
DeviceName string
|
|
IsolationGroup string
|
|
}
|
|
|
|
// Classify maps one PCI function to a device record, or reports false when the
|
|
// function is not something we would ever schedule.
|
|
func Classify(in Input) (model.Device, bool) {
|
|
vend := norm(in.Vendor)
|
|
dev := norm(in.Device)
|
|
class := strings.ToLower(in.Class)
|
|
|
|
out := model.Device{
|
|
NativeID: in.NativeID,
|
|
Vendor: vendorName(vend, in.VendorName),
|
|
Model: strings.TrimSpace(in.DeviceName),
|
|
IsolationGroup: in.IsolationGroup,
|
|
}
|
|
|
|
// Tier 2 first: a curated entry is more specific than a class code, and is
|
|
// the only thing that can correct a misleading class.
|
|
for _, e := range table {
|
|
if e.Vendor == vend && e.Device == dev {
|
|
out.Capability = e.Capability
|
|
out.Confidence = model.ConfMap
|
|
return out, true
|
|
}
|
|
}
|
|
|
|
// Tier 1: class codes, reliable for accelerators and storage.
|
|
switch {
|
|
case hasClass(class, classVGA), hasClass(class, class3D):
|
|
out.Capability = model.CapGPU
|
|
out.Confidence = model.ConfClass
|
|
return out, true
|
|
case hasClass(class, classNVMe):
|
|
out.Capability = model.CapNVMe
|
|
out.Confidence = model.ConfClass
|
|
return out, true
|
|
case hasClass(class, classInfiniBand):
|
|
// The card is in IB mode right now, which is stronger evidence than
|
|
// the map gives - but still not proof the fabric is up, so it stays
|
|
// below agent confidence and will not be scheduled on yet.
|
|
out.Capability = model.CapInfiniBand
|
|
out.Confidence = model.ConfClass
|
|
return out, true
|
|
case hasClass(class, classEthernet):
|
|
// Plain Ethernet is not a schedulable capability, and guessing RoCE
|
|
// from the class would be exactly the false positive this package
|
|
// exists to prevent.
|
|
return model.Device{}, false
|
|
}
|
|
return model.Device{}, false
|
|
}
|
|
|
|
// LookupNote returns the curated table's justification for a device, for
|
|
// operator-facing output.
|
|
func LookupNote(vendor, device string) (string, bool) {
|
|
v, d := norm(vendor), norm(device)
|
|
for _, e := range table {
|
|
if e.Vendor == v && e.Device == d {
|
|
return e.Note, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// Table exposes the curated rows so they can be listed and reviewed.
|
|
func Table() []Entry { return append([]Entry(nil), table...) }
|
|
|
|
func hasClass(full, want string) bool {
|
|
return strings.HasPrefix(full, want)
|
|
}
|
|
|
|
func norm(s string) string {
|
|
s = strings.ToLower(strings.TrimSpace(s))
|
|
s = strings.TrimPrefix(s, "0x")
|
|
return s
|
|
}
|
|
|
|
func vendorName(id, given string) string {
|
|
if n, ok := vendors[id]; ok {
|
|
return n
|
|
}
|
|
if given != "" {
|
|
// "Mellanox Technologies" -> "mellanox"
|
|
f := strings.Fields(given)
|
|
if len(f) > 0 {
|
|
return strings.ToLower(f[0])
|
|
}
|
|
}
|
|
return id
|
|
}
|