llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
157 lines
4.8 KiB
Go
157 lines
4.8 KiB
Go
package engines
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func llamaSpec(gpus []int) ShardSpec {
|
|
return ShardSpec{
|
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
|
|
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
|
|
Profile: Profile{
|
|
Name: "cpu", Engine: "llamacpp", TP: 1,
|
|
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppCommand(t *testing.T) {
|
|
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cmds) != 1 {
|
|
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
|
|
}
|
|
c := cmds[0]
|
|
if c.Path != "llama-server" {
|
|
t.Errorf("path %q", c.Path)
|
|
}
|
|
|
|
// --metrics is not optional: llama-server serves no /metrics without it, and
|
|
// an endpoint the router cannot measure is one it routes to blind.
|
|
if !slices.Contains(c.Args, "--metrics") {
|
|
t.Error("the server was started without --metrics")
|
|
}
|
|
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
|
|
t.Errorf("--model %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
|
|
t.Errorf("--port %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
|
|
t.Errorf("--alias %q", v)
|
|
}
|
|
// llama.cpp serves one request at a time unless told otherwise, which would
|
|
// make every second request wait no matter what the router decided.
|
|
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
if slices.Contains(c.Args, "--n-gpu-layers") {
|
|
t.Error("a CPU placement was given GPU layers")
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
|
|
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
|
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
|
|
t.Errorf("a GPU placement should offload the model, got %q", v)
|
|
}
|
|
// The shard must still be pinned to its own card.
|
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
|
|
t.Error("the shard was not pinned to its GPU")
|
|
}
|
|
}
|
|
|
|
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
|
|
// The escape hatch has to actually work, or an operator tuning slot count
|
|
// ends up with two --parallel flags and whichever llama.cpp happens to
|
|
// prefer.
|
|
s := llamaSpec(nil)
|
|
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
n := 0
|
|
for _, a := range cmds[0].Args {
|
|
if a == "--parallel" {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("--parallel appears %d times", n)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
|
|
t.Errorf("the profile's value lost: %q", v)
|
|
}
|
|
}
|
|
|
|
func TestShortFormArgumentsAreRecognized(t *testing.T) {
|
|
s := llamaSpec([]int{0})
|
|
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
|
|
if slices.Contains(cmds[0].Args, flag) {
|
|
t.Errorf("%s was added although the profile set its short form", flag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
|
|
// Accepting either of these would produce a replica that starts and then
|
|
// serves from one machine while the catalog believes it has several.
|
|
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
|
|
t.Error("a multi-node llama.cpp profile was accepted")
|
|
}
|
|
|
|
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
|
|
t.Error("a tensor-parallel llama.cpp profile was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileValidates(t *testing.T) {
|
|
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
|
|
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
|
|
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
|
|
// Unlike the others there is no worker rank to get this wrong for.
|
|
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
|
|
t.Error("a llama.cpp shard must answer the API")
|
|
}
|
|
}
|
|
|
|
func TestBinaryIsConfigurable(t *testing.T) {
|
|
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
|
|
cmds, _ := l.Commands(llamaSpec(nil))
|
|
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
|
|
t.Errorf("path %q", cmds[0].Path)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
}
|
|
|
|
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
|
|
_, err := NewRegistry(Config{}).Get("llama.cpp")
|
|
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
|
|
t.Errorf("the suggestion should name the real engine: %v", err)
|
|
}
|
|
}
|
|
|
|
// flagValue reads a "--flag value" pair out of a command line.
|
|
func flagValue(args []string, flag string) (string, bool) {
|
|
for i, a := range args {
|
|
if a == flag && i+1 < len(args) {
|
|
return args[i+1], true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|