package engines import ( "os" "path/filepath" "slices" "strings" "testing" ) func llamaSpec(gpus []int) ShardSpec { return ShardSpec{ ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1, GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0", Profile: Profile{ Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b", }, } } func TestLlamaCppCommand(t *testing.T) { cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil)) if err != nil { t.Fatal(err) } if len(cmds) != 1 { t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds)) } c := cmds[0] if c.Path != "llama-server" { t.Errorf("path %q", c.Path) } // --metrics is not optional: llama-server serves no /metrics without it, and // an endpoint the router cannot measure is one it routes to blind. if !slices.Contains(c.Args, "--metrics") { t.Error("the server was started without --metrics") } if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" { t.Errorf("--model %q", v) } if v, _ := flagValue(c.Args, "--port"); v != "8000" { t.Errorf("--port %q", v) } if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" { t.Errorf("--alias %q", v) } // llama.cpp serves one request at a time unless told otherwise, which would // make every second request wait no matter what the router decided. if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" { t.Errorf("--parallel %q", v) } if slices.Contains(c.Args, "--n-gpu-layers") { t.Error("a CPU placement was given GPU layers") } } func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) { cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" { t.Errorf("a GPU placement should offload the model, got %q", v) } // The shard must still be pinned to its own card. if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") { t.Error("the shard was not pinned to its GPU") } } func TestProfileArgumentsWinOverDefaults(t *testing.T) { // The escape hatch has to actually work, or an operator tuning slot count // ends up with two --parallel flags and whichever llama.cpp happens to // prefer. s := llamaSpec(nil) s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"} cmds, _ := (&LlamaCpp{}).Commands(s) n := 0 for _, a := range cmds[0].Args { if a == "--parallel" { n++ } } if n != 1 { t.Errorf("--parallel appears %d times", n) } if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" { t.Errorf("the profile's value lost: %q", v) } } func TestShortFormArgumentsAreRecognized(t *testing.T) { s := llamaSpec([]int{0}) s.Profile.Args = []string{"-np", "8", "-ngl", "20"} cmds, _ := (&LlamaCpp{}).Commands(s) for _, flag := range []string{"--parallel", "--n-gpu-layers"} { if slices.Contains(cmds[0].Args, flag) { t.Errorf("%s was added although the profile set its short form", flag) } } } func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) { // Accepting either of these would produce a replica that starts and then // serves from one machine while the catalog believes it has several. multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"} if err := (&LlamaCpp{}).Validate(multi, nil); err == nil { t.Error("a multi-node llama.cpp profile was accepted") } tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"} if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil { t.Error("a tensor-parallel llama.cpp profile was accepted") } } func TestCPUProfileValidates(t *testing.T) { p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"} if err := (&LlamaCpp{}).Validate(p, nil); err != nil { t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err) } } func TestLlamaCppAlwaysServesHTTP(t *testing.T) { // Unlike the others there is no worker rank to get this wrong for. if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) { t.Error("a llama.cpp shard must answer the API") } } func TestBinaryIsConfigurable(t *testing.T) { l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}} cmds, _ := l.Commands(llamaSpec(nil)) if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" { t.Errorf("path %q", cmds[0].Path) } if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" { t.Errorf("--parallel %q", v) } } func TestUnknownEngineNamesLlamaCpp(t *testing.T) { _, err := NewRegistry(Config{}).Get("llama.cpp") if err == nil || !strings.Contains(err.Error(), "llamacpp") { t.Errorf("the suggestion should name the real engine: %v", err) } } // flagValue reads a "--flag value" pair out of a command line. func flagValue(args []string, flag string) (string, bool) { for i, a := range args { if a == flag && i+1 < len(args) { return args[i+1], true } } return "", false } func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) { // Pinned so an engine upgrade cannot change how prompts are built under a // running fleet: build 10991 defaults to jinja, older builds do not. // // Not a fix for tool calling, though it was added while chasing that. On // this build the flag is a no-op and the behaviour is identical with and // without it. l := &LlamaCpp{} cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}}) if err != nil { t.Fatal(err) } if !slices.Contains(cmds[0].Args, "--jinja") { t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args) } } func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) { // A model whose bundled template is broken has to be serviceable without // editing the adapter. l := &LlamaCpp{} cmds, err := l.Commands(ShardSpec{ HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}}, }) if err != nil { t.Fatal(err) } if slices.Contains(cmds[0].Args, "--jinja") { t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args) } } func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) { dir := t.TempDir() // Deliberately out of order on disk, because readdir order is not shard // order and the first part is the one llama.cpp must be given. for _, n := range []string{ "qwen-q4_k_m-00003-of-00003.gguf", "qwen-q4_k_m-00001-of-00003.gguf", "qwen-q4_k_m-00002-of-00003.gguf", } { if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { t.Fatal(err) } } got, _, err := ggufIn(dir) if err != nil { t.Fatal(err) } // Given the first part llama.cpp finds the rest beside it; given the third // it loads a third of a model. if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" { t.Errorf("expected the first shard, got %s", filepath.Base(got)) } } func TestAPlainFilePathIsUntouched(t *testing.T) { // Every single-file model and every hand-staged one takes this path. p := filepath.Join(t.TempDir(), "model.gguf") if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { t.Fatal(err) } if got, _, _ := ggufIn(p); got != p { t.Errorf("expected the path unchanged, got %s", got) } } func TestADirectoryWithNoWeightsSaysSo(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil { t.Fatal(err) } if _, _, err := ggufIn(dir); err == nil { t.Error("a directory with no .gguf in it is not a model llama.cpp can load") } } func TestAMissingPathIsLeftToTheEngine(t *testing.T) { // The engine's own "no such file" names the path and the flag it came // from, which is more use than anything this could say. p := "/nonexistent/model.gguf" got, _, err := ggufIn(p) if err != nil || got != p { t.Errorf("expected it passed through, got %q %v", got, err) } } func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) { // The command line is readable by every user on the node; the // environment of a root process is not. cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1", Profile: Profile{ModelPath: "/m.gguf"}}) if err != nil { t.Fatal(err) } if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") { t.Errorf("the key is on the command line: %v", cmds[0].Args) } if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") { t.Error("llama-server was not given its key") } } func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) { prev := numCPU numCPU = func() int { return 8 } t.Cleanup(func() { numCPU = prev }) threads := func(l *LlamaCpp, s ShardSpec) (string, bool) { cmds, err := l.Commands(s) if err != nil { t.Fatal(err) } return flagValue(cmds[0].Args, "--threads") } if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" { t.Errorf("--threads %q on 8 CPUs, expected 4", v) } if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" { t.Errorf("--threads %q with 8 configured", v) } if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok { t.Error("a GPU shard was given a thread count") } s := llamaSpec(nil) s.Profile.Args = []string{"-t", "2"} cmds, _ := (&LlamaCpp{}).Commands(s) if slices.Contains(cmds[0].Args, "--threads") { t.Error("the profile's own -t was overridden") } numCPU = func() int { return 1 } if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" { t.Errorf("--threads %q on one CPU", v) } } func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) { s := llamaSpec(nil) s.Threads = 3 cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s) if err != nil { t.Fatal(err) } if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" { t.Errorf("--threads %q; what placement reserved should win over the node's setting", v) } } func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { s := llamaSpec(nil) s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}} cmds, err := (&LlamaCpp{}).Commands(s) if err != nil { t.Fatal(err) } args := strings.Join(cmds[0].Args, " ") if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") { t.Errorf("args %s: both adapters, in order, loaded at scale zero", args) } plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil)) if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") { t.Error("a model with no adapters was given lora flags") } } func TestDevicesAreReadFromListDevices(t *testing.T) { cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") { t.Errorf("CUDA build: %q", got) } // A CUDA build whose backend did not load says what the CPU build says. if got := parseDevices("Available devices:\n (none)\n"); len(got) != 0 { t.Errorf("no devices: %q", got) } } func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) { four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"} if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil { t.Fatalf("four GPUs on a four-GPU node: %v", err) } if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil { t.Error("four GPUs were accepted on a node with two") } s := llamaSpec([]int{0, 1, 2, 3}) s.Profile.GPUsPerNode = 4 cmds, err := (&LlamaCpp{}).Commands(s) if err != nil { t.Fatal(err) } if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" { t.Errorf("--split-mode %q over four GPUs", v) } if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") { t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env) } one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) if hasFlag(one[0].Args, "--split-mode") { t.Error("a one-GPU replica was given a split mode") } } func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) { dir := t.TempDir() // "mmproj" sorts before the model's name; it must still not be taken for it. for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} { if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { t.Fatal(err) } } cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}}) if err != nil { t.Fatal(err) } args := strings.Join(cmds[0].Args, " ") if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) || !strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) { t.Errorf("%s", args) } // One the profile names itself is left as it is. cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir, Args: []string{"--mmproj", "/elsewhere.gguf"}}}) if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 { t.Errorf("two projectors: %v", cmds[0].Args) } }