Tool calling did not work, and it looked like the model's fault. Asked to
use a tool, qwen2.5-coder produced exactly the right call —
{"name": "list_files", "arguments": {"path": "."}} — as plain reply text,
with tool_calls null and finish_reason "stop". Every OpenAI-compatible
client, opencode among them, sees a model that ignores the tools it was
given.
The cause is that llama-server falls back to a generic chat template unless
--jinja is passed, and the generic one neither formats the tool definitions
the way the model was trained to read them nor parses what comes back into
tool_calls. It affects ordinary chat too, for the same reason.
A profile can still pass --no-jinja, because a model whose bundled template
is broken has to be serviceable without editing the adapter.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
129 lines
4.6 KiB
Go
129 lines
4.6 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// LlamaCppConfig locates the llama.cpp installation.
|
|
type LlamaCppConfig struct {
|
|
// Bin is the llama-server binary. It is the server, not the CLI: the
|
|
// gateway talks OpenAI over HTTP and needs the one that serves it.
|
|
Bin string `koanf:"bin"`
|
|
// Parallel is how many requests a server handles at once, when a profile
|
|
// does not say. llama.cpp defaults to one, which would make every second
|
|
// request wait behind the first no matter what the router decided.
|
|
Parallel int `koanf:"parallel"`
|
|
}
|
|
|
|
// LlamaCpp builds llama.cpp launch commands.
|
|
//
|
|
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
|
// small model on CPU at a speed that is fine for testing the whole path, and it
|
|
// serves the same OpenAI API as the others, so nothing downstream can tell the
|
|
// difference.
|
|
type LlamaCpp struct{ cfg LlamaCppConfig }
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*LlamaCpp) Name() string { return "llamacpp" }
|
|
|
|
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
|
|
// is still loading, which is exactly what the supervisor needs.
|
|
func (*LlamaCpp) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
|
|
// server was started with --metrics. Commands always passes it.
|
|
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
|
|
|
|
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
|
|
// does, because a llama.cpp replica is always exactly one shard.
|
|
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
//
|
|
// The two refusals here are about not pretending: llama.cpp has no equivalent
|
|
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
|
|
// different mechanism (--split-mode, --tensor-split) that this adapter does not
|
|
// drive. Accepting either would produce a replica that starts and then serves
|
|
// from one machine while the catalog believes it has several.
|
|
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
p.Name, p.Nodes)
|
|
}
|
|
if p.TP > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
|
|
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
|
|
p.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Commands returns the processes for one shard, which is always one process.
|
|
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
|
bin := l.cfg.Bin
|
|
if bin == "" {
|
|
bin = "llama-server"
|
|
}
|
|
p := s.Profile
|
|
|
|
args := []string{
|
|
"--model", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
// Without this there is no /metrics, and an endpoint the router cannot
|
|
// measure is one it has to route to blind.
|
|
"--metrics",
|
|
}
|
|
// The model's own chat template, which is what turns a tool call into a
|
|
// tool_calls array instead of a JSON object sitting in the reply text.
|
|
//
|
|
// Without it qwen2.5-coder emitted exactly the right call —
|
|
// {"name": "list_files", "arguments": {"path": "."}} — as plain content,
|
|
// and every OpenAI-compatible client saw a model that ignores its tools.
|
|
// It also affects ordinary chat, because the generic fallback template is
|
|
// not the one the model was trained on.
|
|
if !hasFlag(p.Args, "--jinja", "--no-jinja") {
|
|
args = append(args, "--jinja")
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--alias", p.ServedModelName)
|
|
}
|
|
if !hasFlag(p.Args, "--parallel", "-np") {
|
|
n := l.cfg.Parallel
|
|
if n <= 0 {
|
|
n = 4
|
|
}
|
|
args = append(args, "--parallel", strconv.Itoa(n))
|
|
}
|
|
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
|
|
// CPU otherwise, which is not what a GPU placement asked for. Without them
|
|
// the flag is omitted entirely rather than set to zero, so a build with no
|
|
// GPU support does not see an argument it rejects.
|
|
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
|
args = append(args, "--n-gpu-layers", "999")
|
|
}
|
|
args = append(args, p.Args...)
|
|
|
|
return []Command{{
|
|
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
|
}}, nil
|
|
}
|
|
|
|
// hasFlag reports whether the profile already sets one of these flags, in
|
|
// either "--flag value" or "--flag=value" form.
|
|
func hasFlag(args []string, names ...string) bool {
|
|
for _, a := range args {
|
|
for _, n := range names {
|
|
if a == n || strings.HasPrefix(a, n+"=") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|