Export what the cluster is made of, not only what it is serving
CI / test (push) Successful in 1m28s
CI / test (push) Successful in 1m28s
The router published twenty-two metric families about requests, endpoints, admission and scoring — everything about traffic and nothing about the machines carrying it. There was no way to graph GPU usage, watch a node go unhealthy, see a fabric fill up, or answer "was the cluster full at the time", which is the first question anyone asks after an incident. The agents exported nothing at all. Exported by the control plane rather than by each agent. It already holds every number: hardware arrives at registration and state on each heartbeat, so this is a map read away. One scrape target for a fleet of hundreds, no Prometheus configuration per node, no firewall rule per machine, and nothing new running on the nodes. On the same /metrics as the traffic figures, because correlating "the cluster was full" with "requests were queueing" should not need two systems. A collector rather than gauges kept current by a ticker: the numbers are derived from state that already exists, so computing them when asked is simpler than maintaining a copy that can drift, and nothing runs between scrapes. GPU utilisation is new, and is the number people actually watch. Memory used says a model is loaded; this says whether it is doing anything, and a fleet at ninety percent occupancy and two percent utilisation is a fleet holding models nobody is asking for. Adding it caught two things. utilization.gpu goes last in the nvidia-smi query and is optional, because a driver too old to know it returns a shorter row — requiring it made an llmgw upgrade silently stop discovering GPUs on an older node, which is far worse than not knowing how busy a card is. And I gave the new protobuf field number 14, which ecc_errors already had; buf lint is why that was a moment rather than a corrupted wire format. Absent is kept distinct from zero throughout, for the same reason as everywhere else here: an unreported utilisation shown as a flat zero reads as an idle fleet rather than a missing measurement, and a replica nothing has reported would be the most idle thing in the cluster. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
943c23a412
commit
42fd0aa31e
12 files changed
+488
-33
No files matched your search
@@ -20,6 +20,14 @@ type GPU struct {
|
||||
// MemoryUsedMB is what is already allocated. Non-zero on an idle node means
|
||||
// something other than our engines is using the card (plan §5.1).
|
||||
MemoryUsedMB int `json:"memory_used_mb"`
|
||||
// UtilizationPercent is how busy the card was over the last sample period.
|
||||
//
|
||||
// The number people actually watch, and the one nothing collected: memory
|
||||
// used says a model is loaded, and this says whether it is doing anything.
|
||||
// A fleet of cards at ninety percent occupancy and two percent utilisation
|
||||
// is a fleet holding models nobody is asking for, which is invisible
|
||||
// without it.
|
||||
UtilizationPercent int `json:"utilization_percent,omitempty"`
|
||||
// ComputeCapability gates quantization choices: an FP8 profile must not
|
||||
// land on a card below 8.9 (plan §6.2).
|
||||
ComputeCapability string `json:"compute_capability"`
|
||||
|
||||
@@ -21,8 +21,21 @@ var smiQueryFields = []string{
|
||||
"driver_version",
|
||||
"temperature.gpu",
|
||||
"ecc.errors.uncorrected.volatile.total",
|
||||
// Last, and optional. An nvidia-smi too old to know this field returns a
|
||||
// shorter row, and every column before it still lines up — which is why it
|
||||
// goes on the end rather than anywhere else, and why the parser requires
|
||||
// only the mandatory prefix.
|
||||
"utilization.gpu",
|
||||
}
|
||||
|
||||
// mandatorySMIFields is how many columns a row must have to be usable.
|
||||
//
|
||||
// Everything before utilization.gpu. A driver that does not know the last field
|
||||
// gives a short row, and rejecting those would make an llmgw upgrade silently
|
||||
// stop discovering GPUs on an older node — a far worse outcome than not knowing
|
||||
// how busy a card is.
|
||||
const mandatorySMIFields = 10
|
||||
|
||||
// DiscoverGPUsSMI enumerates GPUs by shelling out to nvidia-smi.
|
||||
//
|
||||
// This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
|
||||
@@ -55,7 +68,7 @@ func parseSMI(out string) ([]GPU, error) {
|
||||
|
||||
gpus := make([]GPU, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < len(smiQueryFields) {
|
||||
if len(row) < mandatorySMIFields {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
||||
@@ -76,6 +89,9 @@ func parseSMI(out string) ([]GPU, error) {
|
||||
g.MemoryUsedMB, _ = parseCSVInt(row[4])
|
||||
g.Health.TemperatureC, _ = parseCSVInt(row[8])
|
||||
g.Health.ECCErrors, _ = parseCSVInt(row[9])
|
||||
if len(row) > 10 {
|
||||
g.UtilizationPercent, _ = parseCSVInt(row[10])
|
||||
}
|
||||
|
||||
assessHealth(&g)
|
||||
gpus = append(gpus, g)
|
||||
|
||||
@@ -142,3 +142,38 @@ func TestDiscoverGPUsSMIMissingBinary(t *testing.T) {
|
||||
t.Error("expected an error when nvidia-smi is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderDriverWithoutUtilisationStillDiscovers(t *testing.T) {
|
||||
// An nvidia-smi too old for utilization.gpu returns a shorter row. Rejecting
|
||||
// those would make an llmgw upgrade silently stop discovering GPUs on an
|
||||
// older node, which is far worse than not knowing how busy a card is — and
|
||||
// is what the first version of this change did.
|
||||
old := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0"
|
||||
gpus, err := parseSMI(old)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("a row without utilisation produced %d GPUs", len(gpus))
|
||||
}
|
||||
if gpus[0].MemoryTotalMB != 81559 || gpus[0].Health.TemperatureC != 41 {
|
||||
t.Errorf("columns shifted: %+v", gpus[0])
|
||||
}
|
||||
if gpus[0].UtilizationPercent != 0 {
|
||||
t.Errorf("utilisation = %d, want 0 when unreported", gpus[0].UtilizationPercent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUtilisationIsReadWhenPresent(t *testing.T) {
|
||||
row := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0, 87"
|
||||
gpus, err := parseSMI(row)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("got %d GPUs", len(gpus))
|
||||
}
|
||||
if gpus[0].UtilizationPercent != 87 {
|
||||
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user