llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
92 lines
3.1 KiB
Go
92 lines
3.1 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
)
|
|
|
|
// fallbackEngine is an engine that comes up on the CPU when its GPU backend
|
|
// does not load, and lists the devices it can see.
|
|
type fallbackEngine struct {
|
|
engines.Engine
|
|
devices []string
|
|
}
|
|
|
|
func (fallbackEngine) FallsBackToCPU() bool { return true }
|
|
func (f fallbackEngine) Devices(context.Context) ([]string, error) {
|
|
return f.devices, nil
|
|
}
|
|
|
|
func TestAGPUShardComputingOnTheCPUIsNotReady(t *testing.T) {
|
|
procs := map[int][]int{}
|
|
oldProcs, oldGroup, oldWait := gpuProcesses, processGroup, gpuCheckWait
|
|
gpuProcesses = func(context.Context, string) (map[int][]int, error) { return procs, nil }
|
|
processGroup = func(pid int) int {
|
|
if pid >= 500 && pid < 600 {
|
|
return 500 // the shard's group: 500 and its children
|
|
}
|
|
return pid
|
|
}
|
|
gpuCheckWait = 0
|
|
t.Cleanup(func() { gpuProcesses, processGroup, gpuCheckWait = oldProcs, oldGroup, oldWait })
|
|
|
|
a := &Agent{log: slog.Default()}
|
|
spec := engines.ShardSpec{ShardID: "s", GPUIndices: []int{0, 1}}
|
|
eng := fallbackEngine{}
|
|
|
|
// On no GPU at all: refused, saying what the engine sees.
|
|
err := a.onItsGPUs(context.Background(), eng, spec, 500)
|
|
if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || !strings.Contains(err.Error(), "lists no GPU") {
|
|
t.Fatalf("a shard on no GPU was let through: %v", err)
|
|
}
|
|
// On one of its two: refused, naming the one it is missing.
|
|
procs[0] = []int{500}
|
|
if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil || !strings.Contains(err.Error(), "GPU 1 ") {
|
|
t.Fatalf("a shard on half its GPUs: %v", err)
|
|
}
|
|
// Another process on GPU 1 is not the shard.
|
|
procs[1] = []int{900}
|
|
if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil {
|
|
t.Fatal("another process's memory was taken for the shard's")
|
|
}
|
|
// A child of the shard holding GPU 1 is the shard.
|
|
procs[1] = []int{900, 512}
|
|
if err := a.onItsGPUs(context.Background(), eng, spec, 500); err != nil {
|
|
t.Fatalf("a shard on both its GPUs: %v", err)
|
|
}
|
|
|
|
// An engine that fails rather than falling back is not checked, nor is a
|
|
// shard with no GPUs.
|
|
procs = map[int][]int{}
|
|
if err := a.onItsGPUs(context.Background(), struct{ engines.Engine }{}, spec, 500); err != nil {
|
|
t.Errorf("checked an engine that does not fall back: %v", err)
|
|
}
|
|
if err := a.onItsGPUs(context.Background(), eng, engines.ShardSpec{}, 500); err != nil {
|
|
t.Errorf("checked a CPU shard: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
|
|
calls := 0
|
|
oldProcs, oldWait := gpuProcesses, gpuCheckWait
|
|
gpuProcesses = func(context.Context, string) (map[int][]int, error) {
|
|
calls++
|
|
if calls < 2 {
|
|
return map[int][]int{}, nil
|
|
}
|
|
return map[int][]int{0: {77}}, nil
|
|
}
|
|
gpuCheckWait = 5 * time.Second
|
|
t.Cleanup(func() { gpuProcesses, gpuCheckWait = oldProcs, oldWait })
|
|
|
|
a := &Agent{log: slog.Default()}
|
|
if err := a.onItsGPUs(context.Background(), fallbackEngine{}, engines.ShardSpec{GPUIndices: []int{0}}, 77); err != nil {
|
|
t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
|
|
}
|
|
}
|