package engines import "testing" const migUUID = "MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f" func TestAShardOnAMIGInstanceIsGivenItByUUID(t *testing.T) { // Placement numbers card 0's instance 1 as 1001, which CUDA has never // heard of: CUDA_VISIBLE_DEVICES=1001 is no device at all, and =1 would // be the wrong card. s := singleNodeSpec() s.GPUIndices, s.GPUUUIDs = []int{1001}, []string{migUUID} s.Profile.TP = 1 env := envMap(buildOne(t, &VLLM{}, s).Env) if env["CUDA_VISIBLE_DEVICES"] != migUUID { t.Errorf("vLLM: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"]) } l := llamaSpec([]int{1001}) l.GPUUUIDs = s.GPUUUIDs cmds, err := (&LlamaCpp{}).Commands(l) if err != nil { t.Fatal(err) } env = envMap(cmds[0].Env) if env["CUDA_VISIBLE_DEVICES"] != migUUID { t.Errorf("llama.cpp: CUDA_VISIBLE_DEVICES=%q, want the instance's UUID", env["CUDA_VISIBLE_DEVICES"]) } // Vulkan does not see MIG instances, and an index for it would name // some other card. if v, set := env["GGML_VK_VISIBLE_DEVICES"]; set { t.Errorf("GGML_VK_VISIBLE_DEVICES=%q on a MIG instance", v) } if v, _ := flagValue(cmds[0].Args, "--n-gpu-layers"); v != "999" { t.Errorf("the model was not offloaded to the instance: --n-gpu-layers %q", v) } } func TestAShardOnWholeCardsIsStillGivenThemByIndex(t *testing.T) { env := envMap(buildOne(t, &VLLM{}, singleNodeSpec()).Env) if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" { t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"]) } cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) if env := envMap(cmds[0].Env); env["GGML_VK_VISIBLE_DEVICES"] != "2" { t.Errorf("GGML_VK_VISIBLE_DEVICES=%q", env["GGML_VK_VISIBLE_DEVICES"]) } }