package engines import ( "fmt" "os" "path/filepath" "sort" "strconv" "strings" "sync/atomic" ) // LlamaCppConfig locates the llama.cpp installation. type LlamaCppConfig struct { // Bin is the llama-server binary. It is the server, not the CLI: the // gateway talks OpenAI over HTTP and needs the one that serves it. // Empty means the agent installs llama.cpp itself, at the version the // control plane asks for. Setting it opts out of that. Bin string `koanf:"bin"` // Parallel is how many requests a server handles at once, when a profile // does not say. llama.cpp defaults to one, which would make every second // request wait behind the first no matter what the router decided. Parallel int `koanf:"parallel"` } // LlamaCpp builds llama.cpp launch commands. // // It is the engine that makes a cluster without GPUs useful: llama.cpp runs a // small model on CPU at a speed that is fine for testing the whole path, and it // serves the same OpenAI API as the others, so nothing downstream can tell the // difference. type LlamaCpp struct { cfg LlamaCppConfig installed atomic.Pointer[string] } // Name identifies the engine in the catalog. func (*LlamaCpp) Name() string { return "llamacpp" } // HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model // is still loading, which is exactly what the supervisor needs. func (*LlamaCpp) HealthPath() string { return "/health" } // MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the // server was started with --metrics. Commands always passes it. func (*LlamaCpp) MetricsPath() string { return "/metrics" } // ServesHTTP reports whether this rank answers the API. Every llama.cpp shard // does, because a llama.cpp replica is always exactly one shard. func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true } // Validate checks the profile against the GPUs it would run on. // // The two refusals here are about not pretending: llama.cpp has no equivalent // of a tensor-parallel group spanning machines, and its multi-GPU support is a // different mechanism (--split-mode, --tensor-split) that this adapter does not // drive. Accepting either would produce a replica that starts and then serves // from one machine while the catalog believes it has several. func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } if p.Nodes > 1 { return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica", p.Name, p.Nodes) } if p.TP > 1 { return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+ "it splits a model across local GPUs with --tensor-split, which this adapter does not drive", p.Name) } return nil } // Commands returns the processes for one shard, which is always one process. func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { bin := l.bin() p := s.Profile model, err := ggufIn(p.ModelPath) if err != nil { return nil, err } args := []string{ "--model", model, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot // measure is one it has to route to blind. "--metrics", } // The model's own chat template rather than a generic one, which is what // renders tool definitions in the form the model was trained to read. // // Pinned rather than relied on: build 10991 has it on by default, older // builds do not, and an engine upgrade should not be able to change how // prompts are built underneath a running fleet. Passing it explicitly was // a no-op on the build here — it was added while chasing tool calls that // came back as text, and did not fix that, which turned out to be the // model rather than the template. if !hasFlag(p.Args, "--jinja", "--no-jinja") { args = append(args, "--jinja") } if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } if !hasFlag(p.Args, "--parallel", "-np") { n := l.cfg.Parallel if n <= 0 { n = 4 } args = append(args, "--parallel", strconv.Itoa(n)) } // With GPUs assigned, offload everything; llama.cpp keeps the model on the // CPU otherwise, which is not what a GPU placement asked for. Without them // the flag is omitted entirely rather than set to zero, so a build with no // GPU support does not see an argument it rejects. if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") { args = append(args, "--n-gpu-layers", "999") } args = append(args, p.Args...) return []Command{{ Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), }}, nil } // hasFlag reports whether the profile already sets one of these flags, in // either "--flag value" or "--flag=value" form. func hasFlag(args []string, names ...string) bool { for _, a := range args { for _, n := range names { if a == n || strings.HasPrefix(a, n+"=") { return true } } } return false } // ggufIn resolves a model path that may be a directory. // // A multi-file version arrives as the directory it was downloaded into, because // that is the only shape a set can take. llama.cpp wants a file: given the // first part of a split GGUF it finds the rest beside it, so the first part is // what it gets. // // A path that is already a file is returned untouched, which is every // single-file model and every hand-staged one. func ggufIn(path string) (string, error) { st, err := os.Stat(path) if err != nil || !st.IsDir() { // Not a directory, or not there at all — the second is the engine's to // report, with the message it would have given anyway. return path, nil //nolint:nilerr // a missing path is the engine's error to give } entries, err := os.ReadDir(path) if err != nil { return "", err } var names []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { names = append(names, e.Name()) } } if len(names) == 0 { return "", fmt.Errorf("%s holds no .gguf file", path) } // Sorted, so a split model's first part comes first: the parts are named // "-00001-of-00004" and lexical order is shard order. sort.Strings(names) return filepath.Join(path, names[0]), nil } // UseInstalled points the engine at a build the agent installed and checked. // It takes precedence over everything else, because it is the only one that // was seen to run. func (l *LlamaCpp) UseInstalled(path string) { l.installed.Store(&path) } // Configured reports whether the agent's configuration names a binary. A // named binary is the operator's deliberate choice; one merely found on PATH // is not, and is not reported as installed. func (l *LlamaCpp) Configured() bool { return l.cfg.Bin != "" } // bin is the llama-server to run: the installed build, else the configured // one, else whatever PATH has — which is only ever reached by a node not // provisioned by the agent and not told where its engine is. func (l *LlamaCpp) bin() string { if p := l.installed.Load(); p != nil { return *p } if l.cfg.Bin != "" { return l.cfg.Bin } return "llama-server" }