package engines import ( "slices" "strings" "testing" ) func llamaSpec(gpus []int) ShardSpec { return ShardSpec{ ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1, GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0", Profile: Profile{ Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b", }, } } func TestLlamaCppCommand(t *testing.T) { cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil)) if err != nil { t.Fatal(err) } if len(cmds) != 1 { t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds)) } c := cmds[0] if c.Path != "llama-server" { t.Errorf("path %q", c.Path) } // --metrics is not optional: llama-server serves no /metrics without it, and // an endpoint the router cannot measure is one it routes to blind. if !slices.Contains(c.Args, "--metrics") { t.Error("the server was started without --metrics") } if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" { t.Errorf("--model %q", v) } if v, _ := flagValue(c.Args, "--port"); v != "8000" { t.Errorf("--port %q", v) } if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" { t.Errorf("--alias %q", v) } // llama.cpp serves one request at a time unless told otherwise, which would // make every second request wait no matter what the router decided. if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" { t.Errorf("--parallel %q", v) } if slices.Contains(c.Args, "--n-gpu-layers") { t.Error("a CPU placement was given GPU layers") } } func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) { cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" { t.Errorf("a GPU placement should offload the model, got %q", v) } // The shard must still be pinned to its own card. if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") { t.Error("the shard was not pinned to its GPU") } } func TestProfileArgumentsWinOverDefaults(t *testing.T) { // The escape hatch has to actually work, or an operator tuning slot count // ends up with two --parallel flags and whichever llama.cpp happens to // prefer. s := llamaSpec(nil) s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"} cmds, _ := (&LlamaCpp{}).Commands(s) n := 0 for _, a := range cmds[0].Args { if a == "--parallel" { n++ } } if n != 1 { t.Errorf("--parallel appears %d times", n) } if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" { t.Errorf("the profile's value lost: %q", v) } } func TestShortFormArgumentsAreRecognized(t *testing.T) { s := llamaSpec([]int{0}) s.Profile.Args = []string{"-np", "8", "-ngl", "20"} cmds, _ := (&LlamaCpp{}).Commands(s) for _, flag := range []string{"--parallel", "--n-gpu-layers"} { if slices.Contains(cmds[0].Args, flag) { t.Errorf("%s was added although the profile set its short form", flag) } } } func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) { // Accepting either of these would produce a replica that starts and then // serves from one machine while the catalog believes it has several. multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"} if err := (&LlamaCpp{}).Validate(multi, nil); err == nil { t.Error("a multi-node llama.cpp profile was accepted") } tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"} if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil { t.Error("a tensor-parallel llama.cpp profile was accepted") } } func TestCPUProfileValidates(t *testing.T) { p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"} if err := (&LlamaCpp{}).Validate(p, nil); err != nil { t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err) } } func TestLlamaCppAlwaysServesHTTP(t *testing.T) { // Unlike the others there is no worker rank to get this wrong for. if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) { t.Error("a llama.cpp shard must answer the API") } } func TestBinaryIsConfigurable(t *testing.T) { l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}} cmds, _ := l.Commands(llamaSpec(nil)) if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" { t.Errorf("path %q", cmds[0].Path) } if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" { t.Errorf("--parallel %q", v) } } func TestUnknownEngineNamesLlamaCpp(t *testing.T) { _, err := NewRegistry(Config{}).Get("llama.cpp") if err == nil || !strings.Contains(err.Error(), "llamacpp") { t.Errorf("the suggestion should name the real engine: %v", err) } } // flagValue reads a "--flag value" pair out of a command line. func flagValue(args []string, flag string) (string, bool) { for i, a := range args { if a == flag && i+1 < len(args) { return args[i+1], true } } return "", false }