Files
llmgw/internal/agent/engines/llamacpp.go
T
Ilker ManapandClaude Opus 5 9fc098fd5b
CI / test (push) Successful in 1m25s
Start llama.cpp with the model's own chat template
Tool calling did not work, and it looked like the model's fault. Asked to
use a tool, qwen2.5-coder produced exactly the right call —
{"name": "list_files", "arguments": {"path": "."}} — as plain reply text,
with tool_calls null and finish_reason "stop". Every OpenAI-compatible
client, opencode among them, sees a model that ignores the tools it was
given.

The cause is that llama-server falls back to a generic chat template unless
--jinja is passed, and the generic one neither formats the tool definitions
the way the model was trained to read them nor parses what comes back into
tool_calls. It affects ordinary chat too, for the same reason.

A profile can still pass --no-jinja, because a model whose bundled template
is broken has to be serviceable without editing the adapter.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 21:20:55 +02:00

129 lines
4.6 KiB
Go

package engines
import (
"fmt"
"strconv"
"strings"
)
// LlamaCppConfig locates the llama.cpp installation.
type LlamaCppConfig struct {
// Bin is the llama-server binary. It is the server, not the CLI: the
// gateway talks OpenAI over HTTP and needs the one that serves it.
Bin string `koanf:"bin"`
// Parallel is how many requests a server handles at once, when a profile
// does not say. llama.cpp defaults to one, which would make every second
// request wait behind the first no matter what the router decided.
Parallel int `koanf:"parallel"`
}
// LlamaCpp builds llama.cpp launch commands.
//
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
// small model on CPU at a speed that is fine for testing the whole path, and it
// serves the same OpenAI API as the others, so nothing downstream can tell the
// difference.
type LlamaCpp struct{ cfg LlamaCppConfig }
// Name identifies the engine in the catalog.
func (*LlamaCpp) Name() string { return "llamacpp" }
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
// is still loading, which is exactly what the supervisor needs.
func (*LlamaCpp) HealthPath() string { return "/health" }
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
// server was started with --metrics. Commands always passes it.
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
// does, because a llama.cpp replica is always exactly one shard.
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
// Validate checks the profile against the GPUs it would run on.
//
// The two refusals here are about not pretending: llama.cpp has no equivalent
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
// different mechanism (--split-mode, --tensor-split) that this adapter does not
// drive. Accepting either would produce a replica that starts and then serves
// from one machine while the catalog believes it has several.
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
if err := validateCommon(p, gpus); err != nil {
return err
}
if p.Nodes > 1 {
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
p.Name, p.Nodes)
}
if p.TP > 1 {
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
p.Name)
}
return nil
}
// Commands returns the processes for one shard, which is always one process.
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
bin := l.cfg.Bin
if bin == "" {
bin = "llama-server"
}
p := s.Profile
args := []string{
"--model", p.ModelPath,
"--host", "0.0.0.0",
"--port", strconv.Itoa(s.HTTPPort),
// Without this there is no /metrics, and an endpoint the router cannot
// measure is one it has to route to blind.
"--metrics",
}
// The model's own chat template, which is what turns a tool call into a
// tool_calls array instead of a JSON object sitting in the reply text.
//
// Without it qwen2.5-coder emitted exactly the right call —
// {"name": "list_files", "arguments": {"path": "."}} — as plain content,
// and every OpenAI-compatible client saw a model that ignores its tools.
// It also affects ordinary chat, because the generic fallback template is
// not the one the model was trained on.
if !hasFlag(p.Args, "--jinja", "--no-jinja") {
args = append(args, "--jinja")
}
if p.ServedModelName != "" {
args = append(args, "--alias", p.ServedModelName)
}
if !hasFlag(p.Args, "--parallel", "-np") {
n := l.cfg.Parallel
if n <= 0 {
n = 4
}
args = append(args, "--parallel", strconv.Itoa(n))
}
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
// CPU otherwise, which is not what a GPU placement asked for. Without them
// the flag is omitted entirely rather than set to zero, so a build with no
// GPU support does not see an argument it rejects.
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
args = append(args, "--n-gpu-layers", "999")
}
args = append(args, p.Args...)
return []Command{{
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
}}, nil
}
// hasFlag reports whether the profile already sets one of these flags, in
// either "--flag value" or "--flag=value" form.
func hasFlag(args []string, names ...string) bool {
for _, a := range args {
for _, n := range names {
if a == n || strings.HasPrefix(a, n+"=") {
return true
}
}
}
return false
}