Ilker ManapandClaude Opus 5.5
a16097cfe8
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
389 lines
13 KiB
Go
389 lines
13 KiB
Go
package engines
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func llamaSpec(gpus []int) ShardSpec {
|
|
return ShardSpec{
|
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
|
|
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
|
|
Profile: Profile{
|
|
Name: "cpu", Engine: "llamacpp", TP: 1,
|
|
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppCommand(t *testing.T) {
|
|
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cmds) != 1 {
|
|
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
|
|
}
|
|
c := cmds[0]
|
|
if c.Path != "llama-server" {
|
|
t.Errorf("path %q", c.Path)
|
|
}
|
|
|
|
// --metrics is not optional: llama-server serves no /metrics without it, and
|
|
// an endpoint the router cannot measure is one it routes to blind.
|
|
if !slices.Contains(c.Args, "--metrics") {
|
|
t.Error("the server was started without --metrics")
|
|
}
|
|
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
|
|
t.Errorf("--model %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
|
|
t.Errorf("--port %q", v)
|
|
}
|
|
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
|
|
t.Errorf("--alias %q", v)
|
|
}
|
|
// llama.cpp serves one request at a time unless told otherwise, which would
|
|
// make every second request wait no matter what the router decided.
|
|
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
if slices.Contains(c.Args, "--n-gpu-layers") {
|
|
t.Error("a CPU placement was given GPU layers")
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
|
|
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
|
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
|
|
t.Errorf("a GPU placement should offload the model, got %q", v)
|
|
}
|
|
// The shard must still be pinned to its own card.
|
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
|
|
t.Error("the shard was not pinned to its GPU")
|
|
}
|
|
}
|
|
|
|
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
|
|
// The escape hatch has to actually work, or an operator tuning slot count
|
|
// ends up with two --parallel flags and whichever llama.cpp happens to
|
|
// prefer.
|
|
s := llamaSpec(nil)
|
|
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
n := 0
|
|
for _, a := range cmds[0].Args {
|
|
if a == "--parallel" {
|
|
n++
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Errorf("--parallel appears %d times", n)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
|
|
t.Errorf("the profile's value lost: %q", v)
|
|
}
|
|
}
|
|
|
|
func TestShortFormArgumentsAreRecognized(t *testing.T) {
|
|
s := llamaSpec([]int{0})
|
|
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
|
|
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
|
|
if slices.Contains(cmds[0].Args, flag) {
|
|
t.Errorf("%s was added although the profile set its short form", flag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
|
|
// Accepting either of these would produce a replica that starts and then
|
|
// serves from one machine while the catalog believes it has several.
|
|
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
|
|
t.Error("a multi-node llama.cpp profile was accepted")
|
|
}
|
|
|
|
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
|
|
t.Error("a tensor-parallel llama.cpp profile was accepted")
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileValidates(t *testing.T) {
|
|
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
|
|
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
|
|
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
|
|
// Unlike the others there is no worker rank to get this wrong for.
|
|
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
|
|
t.Error("a llama.cpp shard must answer the API")
|
|
}
|
|
}
|
|
|
|
func TestBinaryIsConfigurable(t *testing.T) {
|
|
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
|
|
cmds, _ := l.Commands(llamaSpec(nil))
|
|
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
|
|
t.Errorf("path %q", cmds[0].Path)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
|
|
t.Errorf("--parallel %q", v)
|
|
}
|
|
}
|
|
|
|
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
|
|
_, err := NewRegistry(Config{}).Get("llama.cpp")
|
|
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
|
|
t.Errorf("the suggestion should name the real engine: %v", err)
|
|
}
|
|
}
|
|
|
|
// flagValue reads a "--flag value" pair out of a command line.
|
|
func flagValue(args []string, flag string) (string, bool) {
|
|
for i, a := range args {
|
|
if a == flag && i+1 < len(args) {
|
|
return args[i+1], true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
|
|
// Pinned so an engine upgrade cannot change how prompts are built under a
|
|
// running fleet: build 10991 defaults to jinja, older builds do not.
|
|
//
|
|
// Not a fix for tool calling, though it was added while chasing that. On
|
|
// this build the flag is a no-op and the behaviour is identical with and
|
|
// without it.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
|
|
// A model whose bundled template is broken has to be serviceable without
|
|
// editing the adapter.
|
|
l := &LlamaCpp{}
|
|
cmds, err := l.Commands(ShardSpec{
|
|
HTTPPort: 8080,
|
|
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slices.Contains(cmds[0].Args, "--jinja") {
|
|
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
|
|
}
|
|
}
|
|
|
|
func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
|
|
dir := t.TempDir()
|
|
// Deliberately out of order on disk, because readdir order is not shard
|
|
// order and the first part is the one llama.cpp must be given.
|
|
for _, n := range []string{
|
|
"qwen-q4_k_m-00003-of-00003.gguf",
|
|
"qwen-q4_k_m-00001-of-00003.gguf",
|
|
"qwen-q4_k_m-00002-of-00003.gguf",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
got, _, err := ggufIn(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Given the first part llama.cpp finds the rest beside it; given the third
|
|
// it loads a third of a model.
|
|
if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
|
|
t.Errorf("expected the first shard, got %s", filepath.Base(got))
|
|
}
|
|
}
|
|
|
|
func TestAPlainFilePathIsUntouched(t *testing.T) {
|
|
// Every single-file model and every hand-staged one takes this path.
|
|
p := filepath.Join(t.TempDir(), "model.gguf")
|
|
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _, _ := ggufIn(p); got != p {
|
|
t.Errorf("expected the path unchanged, got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := ggufIn(dir); err == nil {
|
|
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
|
|
}
|
|
}
|
|
|
|
func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
|
|
// The engine's own "no such file" names the path and the flag it came
|
|
// from, which is more use than anything this could say.
|
|
p := "/nonexistent/model.gguf"
|
|
got, _, err := ggufIn(p)
|
|
if err != nil || got != p {
|
|
t.Errorf("expected it passed through, got %q %v", got, err)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
|
|
// The command line is readable by every user on the node; the
|
|
// environment of a root process is not.
|
|
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
|
|
Profile: Profile{ModelPath: "/m.gguf"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
|
|
t.Errorf("the key is on the command line: %v", cmds[0].Args)
|
|
}
|
|
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
|
|
t.Error("llama-server was not given its key")
|
|
}
|
|
}
|
|
|
|
func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
|
|
prev := numCPU
|
|
numCPU = func() int { return 8 }
|
|
t.Cleanup(func() { numCPU = prev })
|
|
|
|
threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
|
|
cmds, err := l.Commands(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return flagValue(cmds[0].Args, "--threads")
|
|
}
|
|
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
|
|
t.Errorf("--threads %q on 8 CPUs, expected 4", v)
|
|
}
|
|
if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
|
|
t.Errorf("--threads %q with 8 configured", v)
|
|
}
|
|
if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
|
|
t.Error("a GPU shard was given a thread count")
|
|
}
|
|
s := llamaSpec(nil)
|
|
s.Profile.Args = []string{"-t", "2"}
|
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
|
if slices.Contains(cmds[0].Args, "--threads") {
|
|
t.Error("the profile's own -t was overridden")
|
|
}
|
|
numCPU = func() int { return 1 }
|
|
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
|
|
t.Errorf("--threads %q on one CPU", v)
|
|
}
|
|
}
|
|
|
|
func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
|
|
s := llamaSpec(nil)
|
|
s.Threads = 3
|
|
cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
|
|
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
|
|
}
|
|
}
|
|
|
|
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
|
|
s := llamaSpec(nil)
|
|
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
|
|
cmds, err := (&LlamaCpp{}).Commands(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
args := strings.Join(cmds[0].Args, " ")
|
|
if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") {
|
|
t.Errorf("args %s: both adapters, in order, loaded at scale zero", args)
|
|
}
|
|
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
|
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
|
|
t.Error("a model with no adapters was given lora flags")
|
|
}
|
|
}
|
|
|
|
func TestDevicesAreReadFromListDevices(t *testing.T) {
|
|
cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
|
if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") {
|
|
t.Errorf("CUDA build: %q", got)
|
|
}
|
|
// A CUDA build whose backend did not load says what the CPU build says.
|
|
if got := parseDevices("Available devices:\n (none)\n"); len(got) != 0 {
|
|
t.Errorf("no devices: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
|
|
four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"}
|
|
if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil {
|
|
t.Fatalf("four GPUs on a four-GPU node: %v", err)
|
|
}
|
|
if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil {
|
|
t.Error("four GPUs were accepted on a node with two")
|
|
}
|
|
|
|
s := llamaSpec([]int{0, 1, 2, 3})
|
|
s.Profile.GPUsPerNode = 4
|
|
cmds, err := (&LlamaCpp{}).Commands(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" {
|
|
t.Errorf("--split-mode %q over four GPUs", v)
|
|
}
|
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") {
|
|
t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env)
|
|
}
|
|
one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
|
if hasFlag(one[0].Args, "--split-mode") {
|
|
t.Error("a one-GPU replica was given a split mode")
|
|
}
|
|
}
|
|
|
|
func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) {
|
|
dir := t.TempDir()
|
|
// "mmproj" sorts before the model's name; it must still not be taken for it.
|
|
for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} {
|
|
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
args := strings.Join(cmds[0].Args, " ")
|
|
if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) ||
|
|
!strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) {
|
|
t.Errorf("%s", args)
|
|
}
|
|
// One the profile names itself is left as it is.
|
|
cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir,
|
|
Args: []string{"--mmproj", "/elsewhere.gguf"}}})
|
|
if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 {
|
|
t.Errorf("two projectors: %v", cmds[0].Args)
|
|
}
|
|
}
|