9fc098f said passing --jinja fixes tool calling. It does not. llama.cpp
build 10991 has jinja enabled by default — "--jinja, --no-jinja ... (default:
enabled)" — so the flag changed nothing, and tool calls came back as text
before and after it in exactly the same way.
The flag is worth keeping for a different reason, and the comment now says
that one: it pins how prompts are built, so an engine upgrade cannot change
it underneath a running fleet. Older builds do default to the generic
template.
What actually happens is that llama.cpp renders the tools correctly — the
prompt from /apply-template carries the <tools> block and the instruction to
reply in <tool_call> tags — and qwen2.5-coder answers with a markdown JSON
block instead. 3B and 7B both, and tool_choice "required" does not constrain
it either. That is the model, not the plumbing, and no flag here will fix
it.
Recording it because a wrong explanation in a commit message is worse than
no explanation: the next person to see tool calls arriving as text would
have checked --jinja, found it already set, and had nowhere to go.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
130 lines
4.7 KiB
Go
130 lines
4.7 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// LlamaCppConfig locates the llama.cpp installation.
|
|
type LlamaCppConfig struct {
|
|
// Bin is the llama-server binary. It is the server, not the CLI: the
|
|
// gateway talks OpenAI over HTTP and needs the one that serves it.
|
|
Bin string `koanf:"bin"`
|
|
// Parallel is how many requests a server handles at once, when a profile
|
|
// does not say. llama.cpp defaults to one, which would make every second
|
|
// request wait behind the first no matter what the router decided.
|
|
Parallel int `koanf:"parallel"`
|
|
}
|
|
|
|
// LlamaCpp builds llama.cpp launch commands.
|
|
//
|
|
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
|
// small model on CPU at a speed that is fine for testing the whole path, and it
|
|
// serves the same OpenAI API as the others, so nothing downstream can tell the
|
|
// difference.
|
|
type LlamaCpp struct{ cfg LlamaCppConfig }
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*LlamaCpp) Name() string { return "llamacpp" }
|
|
|
|
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
|
|
// is still loading, which is exactly what the supervisor needs.
|
|
func (*LlamaCpp) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
|
|
// server was started with --metrics. Commands always passes it.
|
|
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
|
|
|
|
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
|
|
// does, because a llama.cpp replica is always exactly one shard.
|
|
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
//
|
|
// The two refusals here are about not pretending: llama.cpp has no equivalent
|
|
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
|
|
// different mechanism (--split-mode, --tensor-split) that this adapter does not
|
|
// drive. Accepting either would produce a replica that starts and then serves
|
|
// from one machine while the catalog believes it has several.
|
|
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
p.Name, p.Nodes)
|
|
}
|
|
if p.TP > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
|
|
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
|
|
p.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Commands returns the processes for one shard, which is always one process.
|
|
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
|
bin := l.cfg.Bin
|
|
if bin == "" {
|
|
bin = "llama-server"
|
|
}
|
|
p := s.Profile
|
|
|
|
args := []string{
|
|
"--model", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
// Without this there is no /metrics, and an endpoint the router cannot
|
|
// measure is one it has to route to blind.
|
|
"--metrics",
|
|
}
|
|
// The model's own chat template rather than a generic one, which is what
|
|
// renders tool definitions in the form the model was trained to read.
|
|
//
|
|
// Pinned rather than relied on: build 10991 has it on by default, older
|
|
// builds do not, and an engine upgrade should not be able to change how
|
|
// prompts are built underneath a running fleet. Passing it explicitly was
|
|
// a no-op on the build here — it was added while chasing tool calls that
|
|
// came back as text, and did not fix that, which turned out to be the
|
|
// model rather than the template.
|
|
if !hasFlag(p.Args, "--jinja", "--no-jinja") {
|
|
args = append(args, "--jinja")
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--alias", p.ServedModelName)
|
|
}
|
|
if !hasFlag(p.Args, "--parallel", "-np") {
|
|
n := l.cfg.Parallel
|
|
if n <= 0 {
|
|
n = 4
|
|
}
|
|
args = append(args, "--parallel", strconv.Itoa(n))
|
|
}
|
|
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
|
|
// CPU otherwise, which is not what a GPU placement asked for. Without them
|
|
// the flag is omitted entirely rather than set to zero, so a build with no
|
|
// GPU support does not see an argument it rejects.
|
|
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
|
args = append(args, "--n-gpu-layers", "999")
|
|
}
|
|
args = append(args, p.Args...)
|
|
|
|
return []Command{{
|
|
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
|
}}, nil
|
|
}
|
|
|
|
// hasFlag reports whether the profile already sets one of these flags, in
|
|
// either "--flag value" or "--flag=value" form.
|
|
func hasFlag(args []string, names ...string) bool {
|
|
for _, a := range args {
|
|
for _, n := range names {
|
|
if a == n || strings.HasPrefix(a, n+"=") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|