CI / test (push) Successful in 1m32s
Storage will be different at every installation — Lustre, NFS over RDMA, an image exported with SRP or NVMe-oF, local NVMe, an rsync — so llmgw observes it rather than managing it. An agent told which directories to look in reports what it can see; how the files got there is nobody's business here, and a node told nothing reports nothing and behaves exactly as every node did before this existed. That fact buys two things. Placement prefers a node that already holds the model over one that must first move tens of gigabytes, as a preference and never a filter: fetching them is a valid answer, and refusing to place would turn a slow start into no start at all. And ctl status shows what each fabric holds. Per fabric because fabrics are not connected to one another. Weights in ib-a are not weights in ib-b — the only path between them is the management network — so a model seeded into one fabric leaves the rest of the cluster unable to serve it without a slow copy, and nothing said so before now. One shared filer counts the same as twenty local copies, because "can this fabric serve it" is the question being asked. Three distinctions the tests hold to, because each is a way to get this quietly wrong. A node that was never asked to look must rank with one that looked and has nothing, or enabling the feature on a single node would drain every other. An empty directory is not weights, because a half-finished download looks exactly like one and would send placement to a node whose engine then fails to load. And local storage is never mistaken for shared: a filer read as local costs an unnecessary copy, while local read as shared would have the cluster believe a model is everywhere when it is on one node. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
140 lines
4.7 KiB
Go
140 lines
4.7 KiB
Go
package discovery
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// GPU is one accelerator on this node.
|
|
type GPU struct {
|
|
Index int `json:"index"`
|
|
UUID string `json:"uuid"`
|
|
Name string `json:"name"`
|
|
// MemoryTotalMB is the total framebuffer size.
|
|
MemoryTotalMB int `json:"memory_total_mb"`
|
|
// MemoryUsedMB is what is already allocated. Non-zero on an idle node means
|
|
// something other than our engines is using the card (plan §5.1).
|
|
MemoryUsedMB int `json:"memory_used_mb"`
|
|
// ComputeCapability gates quantization choices: an FP8 profile must not
|
|
// land on a card below 8.9 (plan §6.2).
|
|
ComputeCapability string `json:"compute_capability"`
|
|
NUMANode int `json:"numa_node"`
|
|
PCIBusID string `json:"pci_bus_id,omitempty"`
|
|
DriverVersion string `json:"driver_version,omitempty"`
|
|
CUDAVersion string `json:"cuda_version,omitempty"`
|
|
// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
|
|
// a TP group should stay inside one NVLink domain.
|
|
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
|
|
// Health carries anything that should keep the GPU out of placement.
|
|
Health GPUHealth `json:"health"`
|
|
}
|
|
|
|
// GPUHealth records conditions that make a GPU unsafe to schedule onto.
|
|
type GPUHealth struct {
|
|
// ECCErrors is the count of uncorrected ECC errors.
|
|
ECCErrors int `json:"ecc_errors"`
|
|
// TemperatureC is the current die temperature.
|
|
TemperatureC int `json:"temperature_c"`
|
|
// Problems lists human-readable reasons to avoid this GPU.
|
|
Problems []string `json:"problems,omitempty"`
|
|
}
|
|
|
|
// OK reports whether the GPU is fit for placement.
|
|
func (h GPUHealth) OK() bool { return len(h.Problems) == 0 }
|
|
|
|
// nonAlnum collapses everything that is not a letter or digit.
|
|
var nonAlnum = regexp.MustCompile(`[^a-z0-9]+`)
|
|
|
|
// Class derives the GPU class used for placement: same model, same memory.
|
|
//
|
|
// Two cards only belong in the same tensor-parallel group if they are the same
|
|
// class, so the derivation has to be stable across driver versions and naming
|
|
// quirks ("NVIDIA H100 80GB HBM3" and "H100-SXM5-80GB" are the same class).
|
|
func (g GPU) Class() string {
|
|
name := strings.ToLower(g.Name)
|
|
name = strings.TrimPrefix(name, "nvidia ")
|
|
name = strings.TrimPrefix(name, "tesla ")
|
|
|
|
// Keep the model token: the first word that contains a digit, which is how
|
|
// NVIDIA names every accelerator (h100, a100, l40s, rtx4090).
|
|
var model string
|
|
for _, tok := range nonAlnum.Split(name, -1) {
|
|
if tok == "" || !strings.ContainsFunc(tok, func(r rune) bool { return r >= '0' && r <= '9' }) {
|
|
continue
|
|
}
|
|
// Skip memory and form-factor tokens, which are not the model.
|
|
if strings.HasSuffix(tok, "gb") || tok == "80gb" || tok == "sxm4" || tok == "sxm5" || tok == "pcie" {
|
|
continue
|
|
}
|
|
model = tok
|
|
break
|
|
}
|
|
if model == "" {
|
|
model = nonAlnum.ReplaceAllString(name, "-")
|
|
}
|
|
|
|
// Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card.
|
|
gb := int(math.Round(float64(g.MemoryTotalMB) / 1024))
|
|
return fmt.Sprintf("%s-%dg", model, gb)
|
|
}
|
|
|
|
// MarshalJSON adds the derived class, so a consumer of the JSON does not have
|
|
// to reimplement the derivation to know which GPUs are interchangeable.
|
|
func (g GPU) MarshalJSON() ([]byte, error) {
|
|
type alias GPU // prevents recursing back into this method
|
|
return json.Marshal(struct {
|
|
alias
|
|
Class string `json:"class"`
|
|
}{alias(g), g.Class()})
|
|
}
|
|
|
|
// Inventory is everything discovery found on this node.
|
|
type Inventory struct {
|
|
GPUs []GPU `json:"gpus"`
|
|
HCAs []HCA `json:"hcas"`
|
|
// Source names the backend that produced the GPU list, so a consumer can
|
|
// tell "no GPUs" from "discovery did not work".
|
|
Source string `json:"source"`
|
|
// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
|
|
// agent older than this field reports, and is treated as no constraint.
|
|
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
|
// Weights are the model files this node can see. Empty means it was not
|
|
// asked to look.
|
|
Weights []LocalWeights `json:"weights,omitempty"`
|
|
}
|
|
|
|
// Classes returns the distinct GPU classes present, sorted.
|
|
func (inv Inventory) Classes() []string {
|
|
seen := map[string]struct{}{}
|
|
for _, g := range inv.GPUs {
|
|
seen[g.Class()] = struct{}{}
|
|
}
|
|
out := make([]string, 0, len(seen))
|
|
for c := range seen {
|
|
out = append(out, c)
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// parseCSVInt reads an integer from nvidia-smi's CSV output, tolerating the
|
|
// "[N/A]" and "Not Supported" values it emits for unsupported queries.
|
|
func parseCSVInt(s string) (int, bool) {
|
|
s = strings.TrimSpace(s)
|
|
s = strings.TrimSuffix(s, " MiB")
|
|
s = strings.TrimSuffix(s, " W")
|
|
if s == "" || strings.HasPrefix(s, "[") || strings.EqualFold(s, "Not Supported") {
|
|
return 0, false
|
|
}
|
|
v, err := strconv.Atoi(s)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return v, true
|
|
}
|