Files
llmgw/internal/agent/discovery/gpuprocs_test.go
T
Ilker ManapandClaude Opus 5.5 477d0bf19d Judge a card shared with another process every heartbeat, not once
A card with something else on it when the agent started stayed out of
placement after that process let go of it, until the agent restarted. The
agent now sends what processes other than its own engines hold on each card,
and the control plane replaces the registration verdict with it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 18:46:23 +02:00

79 lines
2.7 KiB
Go

package discovery
import (
"reflect"
"testing"
)
func TestGPUProcessesOfEveryKindAreListedByIndex(t *testing.T) {
report := `<?xml version="1.0" ?>
<nvidia_smi_log>
<driver_version>580.65.06</driver_version>
<gpu id="00000000:17:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
<processes></processes>
</gpu>
<gpu id="00000000:65:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
<processes>
<process_info><pid>4100</pid><type>C</type><process_name>llama-server</process_name></process_info>
<process_info><pid>4300</pid><type>G</type><process_name>llama-server</process_name></process_info>
</processes>
</gpu>
<gpu id="00000000:B3:00.0"><processes>
<process_info><pid>4200</pid><type>C+G</type></process_info>
</processes></gpu>
</nvidia_smi_log>`
got, err := parseGPUProcesses([]byte(report))
if err != nil {
t.Fatal(err)
}
// A Vulkan engine is a graphics process ("G") and is on its GPU all the
// same; a compute-only query would have missed it.
want := map[int][]int{1: {4100, 4300}, 2: {4200}}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %v, want %v", got, want)
}
if _, err := parseGPUProcesses([]byte("not xml")); err == nil {
t.Error("a report that does not parse was read as no processes")
}
}
func TestGPUMemoryIsReadPerProcess(t *testing.T) {
report := `<?xml version="1.0" ?>
<nvidia_smi_log>
<gpu id="00000000:01:00.0"><product_name>Quadro RTX 6000</product_name>
<fb_memory_usage><total>24576 MiB</total><used>14514 MiB</used><free>10062 MiB</free></fb_memory_usage>
<processes>
<process_info><pid>71315</pid><type>C</type><used_memory>4932 MiB</used_memory></process_info>
<process_info><pid>4100</pid><type>C</type><used_memory>9500 MiB</used_memory></process_info>
</processes>
</gpu>
<gpu id="00000000:02:00.0">
<fb_memory_usage><used>N/A</used></fb_memory_usage>
<processes></processes>
</gpu>
</nvidia_smi_log>`
got, err := parseGPUMemory([]byte(report))
if err != nil {
t.Fatal(err)
}
want := []GPUMemory{
{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
{},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %+v, want %+v", got, want)
}
}
func TestForeignMemoryBelowTheToleranceIsNoProblem(t *testing.T) {
// A CUDA context that has let go of its model still holds a few hundred
// MiB: ComfyUI after /free held 324.
if p := ForeignMemoryProblem(330); p != "" {
t.Errorf("330 MiB is a problem: %q", p)
}
p := ForeignMemoryProblem(5014)
if !IsForeignMemoryProblem(p) || IsForeignMemoryProblem("2 uncorrected ECC errors") {
t.Errorf("%q is not recognised as the foreign memory problem, or ECC is", p)
}
}