Files
llmgw/internal/agent/engines/llamacpp_test.go
T
Ilker ManapandClaude Opus 5 dfbb7557dd
CI / test (push) Successful in 1m42s
Serve a model made of several files
The mirror would happily hold a four-shard model that nothing could serve. A
catalog entry names one source, the agent fetched one URL, and so a version of
several files had no way to be described — which is every safetensors model,
and every GGUF too large to ship whole.

A source ending in a slash names a version rather than a file. The agent fetches
every file in it into one directory and hands the engine the directory: vLLM
reads the shards and the config out of the folder, and llama.cpp is given the
first part and finds the rest beside it.

The trailing slash is the whole signal. A flag beside the source would be a
second place for the two to disagree, and a URL already distinguishes a
directory from a file everywhere else.

The mirror answers a version with a manifest instead of a file listing. A plain
file server gives HTML there, and something that has to know what a version
consists of would be left parsing it. It is on the byte listener, not the
management API: fetching weights is the one thing every node does, and making
it depend on the administrative port would undo the split that keeps that port
closeable.

Built from the directory rather than a file written beside it. A manifest on
disk is a second record of what is there, and the two disagree the first time
somebody copies a file in by hand — which the mirror explicitly allows.

The whole set or nothing. A directory with three of four shards loads as a
broken model rather than failing to load, so a completion marker is written
last and an interrupted fetch starts the missing files again rather than being
found and served. Files already there at the right size are not fetched twice.

Progress is reported under the version's address, not each file's: the shards
waiting on it asked for the version, and progress under a file's own URL
reaches nobody.

A filename from the manifest is refused if it contains a separator. The mirror
is a place an operator copies files into by hand, so what it reports is not
automatically safe to join onto a path.

An HTML listing where a manifest was expected says so. "invalid character '<'"
sends somebody looking at the wrong thing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 21:07:26 +02:00

248 lines
7.7 KiB
Go

package engines
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
// Pinned so an engine upgrade cannot change how prompts are built under a
// running fleet: build 10991 defaults to jinja, older builds do not.
//
// Not a fix for tool calling, though it was added while chasing that. On
// this build the flag is a no-op and the behaviour is identical with and
// without it.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
}
}
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
// A model whose bundled template is broken has to be serviceable without
// editing the adapter.
l := &LlamaCpp{}
cmds, err := l.Commands(ShardSpec{
HTTPPort: 8080,
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
})
if err != nil {
t.Fatal(err)
}
if slices.Contains(cmds[0].Args, "--jinja") {
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
}
}
func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
dir := t.TempDir()
// Deliberately out of order on disk, because readdir order is not shard
// order and the first part is the one llama.cpp must be given.
for _, n := range []string{
"qwen-q4_k_m-00003-of-00003.gguf",
"qwen-q4_k_m-00001-of-00003.gguf",
"qwen-q4_k_m-00002-of-00003.gguf",
} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
got, err := ggufIn(dir)
if err != nil {
t.Fatal(err)
}
// Given the first part llama.cpp finds the rest beside it; given the third
// it loads a third of a model.
if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
t.Errorf("expected the first shard, got %s", filepath.Base(got))
}
}
func TestAPlainFilePathIsUntouched(t *testing.T) {
// Every single-file model and every hand-staged one takes this path.
p := filepath.Join(t.TempDir(), "model.gguf")
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if got, _ := ggufIn(p); got != p {
t.Errorf("expected the path unchanged, got %s", got)
}
}
func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ggufIn(dir); err == nil {
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
}
}
func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
// The engine's own "no such file" names the path and the flag it came
// from, which is more use than anything this could say.
p := "/nonexistent/model.gguf"
got, err := ggufIn(p)
if err != nil || got != p {
t.Errorf("expected it passed through, got %q %v", got, err)
}
}