package engines import ( "fmt" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" ) // LlamaCppConfig locates the llama.cpp installation. type LlamaCppConfig struct { // Bin is the llama-server binary. It is the server, not the CLI: the // gateway talks OpenAI over HTTP and needs the one that serves it. // Empty looks on PATH, then where release archives are unpacked // (/opt/llama.cpp). Bin string `koanf:"bin"` // Parallel is how many requests a server handles at once, when a profile // does not say. llama.cpp defaults to one, which would make every second // request wait behind the first no matter what the router decided. Parallel int `koanf:"parallel"` } // LlamaCpp builds llama.cpp launch commands. // // It is the engine that makes a cluster without GPUs useful: llama.cpp runs a // small model on CPU at a speed that is fine for testing the whole path, and it // serves the same OpenAI API as the others, so nothing downstream can tell the // difference. type LlamaCpp struct{ cfg LlamaCppConfig } // Name identifies the engine in the catalog. func (*LlamaCpp) Name() string { return "llamacpp" } // HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model // is still loading, which is exactly what the supervisor needs. func (*LlamaCpp) HealthPath() string { return "/health" } // MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the // server was started with --metrics. Commands always passes it. func (*LlamaCpp) MetricsPath() string { return "/metrics" } // ServesHTTP reports whether this rank answers the API. Every llama.cpp shard // does, because a llama.cpp replica is always exactly one shard. func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true } // Validate checks the profile against the GPUs it would run on. // // The two refusals here are about not pretending: llama.cpp has no equivalent // of a tensor-parallel group spanning machines, and its multi-GPU support is a // different mechanism (--split-mode, --tensor-split) that this adapter does not // drive. Accepting either would produce a replica that starts and then serves // from one machine while the catalog believes it has several. func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } if p.Nodes > 1 { return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica", p.Name, p.Nodes) } if p.TP > 1 { return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+ "it splits a model across local GPUs with --tensor-split, which this adapter does not drive", p.Name) } return nil } // Commands returns the processes for one shard, which is always one process. func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { bin := l.bin() p := s.Profile model, err := ggufIn(p.ModelPath) if err != nil { return nil, err } args := []string{ "--model", model, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot // measure is one it has to route to blind. "--metrics", } // The model's own chat template rather than a generic one, which is what // renders tool definitions in the form the model was trained to read. // // Pinned rather than relied on: build 10991 has it on by default, older // builds do not, and an engine upgrade should not be able to change how // prompts are built underneath a running fleet. Passing it explicitly was // a no-op on the build here — it was added while chasing tool calls that // came back as text, and did not fix that, which turned out to be the // model rather than the template. if !hasFlag(p.Args, "--jinja", "--no-jinja") { args = append(args, "--jinja") } if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } if !hasFlag(p.Args, "--parallel", "-np") { n := l.cfg.Parallel if n <= 0 { n = 4 } args = append(args, "--parallel", strconv.Itoa(n)) } // With GPUs assigned, offload everything; llama.cpp keeps the model on the // CPU otherwise, which is not what a GPU placement asked for. Without them // the flag is omitted entirely rather than set to zero, so a build with no // GPU support does not see an argument it rejects. if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") { args = append(args, "--n-gpu-layers", "999") } args = append(args, p.Args...) return []Command{{ Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), }}, nil } // hasFlag reports whether the profile already sets one of these flags, in // either "--flag value" or "--flag=value" form. func hasFlag(args []string, names ...string) bool { for _, a := range args { for _, n := range names { if a == n || strings.HasPrefix(a, n+"=") { return true } } } return false } // ggufIn resolves a model path that may be a directory. // // A multi-file version arrives as the directory it was downloaded into, because // that is the only shape a set can take. llama.cpp wants a file: given the // first part of a split GGUF it finds the rest beside it, so the first part is // what it gets. // // A path that is already a file is returned untouched, which is every // single-file model and every hand-staged one. func ggufIn(path string) (string, error) { st, err := os.Stat(path) if err != nil || !st.IsDir() { // Not a directory, or not there at all — the second is the engine's to // report, with the message it would have given anyway. return path, nil //nolint:nilerr // a missing path is the engine's error to give } entries, err := os.ReadDir(path) if err != nil { return "", err } var names []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { names = append(names, e.Name()) } } if len(names) == 0 { return "", fmt.Errorf("%s holds no .gguf file", path) } // Sorted, so a split model's first part comes first: the parts are named // "-00001-of-00004" and lexical order is shard order. sort.Strings(names) return filepath.Join(path, names[0]), nil } // llamaServerPlaces are where llama.cpp's release archives are unpacked when // they are not on PATH. The archive carries its shared libraries beside the // binary, so it is usually left in one directory rather than installed. var llamaServerPlaces = []string{ "/opt/llama.cpp/llama-server", "/opt/llama.cpp/bin/llama-server", "/opt/llama.cpp/build/bin/llama-server", "/usr/local/llama.cpp/llama-server", } // bin is the llama-server to run: the configured one, else the one on PATH, // else the first of the usual unpack locations that exists. // // Found rather than required in the configuration, because every node here // had the same path written into its file by hand, and a node set up without // it reported no engine at all. func (l *LlamaCpp) bin() string { if l.cfg.Bin != "" { return l.cfg.Bin } if p, err := exec.LookPath("llama-server"); err == nil { return p } for _, p := range llamaServerPlaces { if info, err := os.Stat(p); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { return p } } return "llama-server" }