package engines import ( "fmt" "strconv" "strings" ) // LlamaCppConfig locates the llama.cpp installation. type LlamaCppConfig struct { // Bin is the llama-server binary. It is the server, not the CLI: the // gateway talks OpenAI over HTTP and needs the one that serves it. Bin string `koanf:"bin"` // Parallel is how many requests a server handles at once, when a profile // does not say. llama.cpp defaults to one, which would make every second // request wait behind the first no matter what the router decided. Parallel int `koanf:"parallel"` } // LlamaCpp builds llama.cpp launch commands. // // It is the engine that makes a cluster without GPUs useful: llama.cpp runs a // small model on CPU at a speed that is fine for testing the whole path, and it // serves the same OpenAI API as the others, so nothing downstream can tell the // difference. type LlamaCpp struct{ cfg LlamaCppConfig } // Name identifies the engine in the catalog. func (*LlamaCpp) Name() string { return "llamacpp" } // HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model // is still loading, which is exactly what the supervisor needs. func (*LlamaCpp) HealthPath() string { return "/health" } // MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the // server was started with --metrics. Commands always passes it. func (*LlamaCpp) MetricsPath() string { return "/metrics" } // ServesHTTP reports whether this rank answers the API. Every llama.cpp shard // does, because a llama.cpp replica is always exactly one shard. func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true } // Validate checks the profile against the GPUs it would run on. // // The two refusals here are about not pretending: llama.cpp has no equivalent // of a tensor-parallel group spanning machines, and its multi-GPU support is a // different mechanism (--split-mode, --tensor-split) that this adapter does not // drive. Accepting either would produce a replica that starts and then serves // from one machine while the catalog believes it has several. func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } if p.Nodes > 1 { return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica", p.Name, p.Nodes) } if p.TP > 1 { return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+ "it splits a model across local GPUs with --tensor-split, which this adapter does not drive", p.Name) } return nil } // Commands returns the processes for one shard, which is always one process. func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { bin := l.cfg.Bin if bin == "" { bin = "llama-server" } p := s.Profile args := []string{ "--model", p.ModelPath, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot // measure is one it has to route to blind. "--metrics", } if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } if !hasFlag(p.Args, "--parallel", "-np") { n := l.cfg.Parallel if n <= 0 { n = 4 } args = append(args, "--parallel", strconv.Itoa(n)) } // With GPUs assigned, offload everything; llama.cpp keeps the model on the // CPU otherwise, which is not what a GPU placement asked for. Without them // the flag is omitted entirely rather than set to zero, so a build with no // GPU support does not see an argument it rejects. if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") { args = append(args, "--n-gpu-layers", "999") } args = append(args, p.Args...) return []Command{{ Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), }}, nil } // hasFlag reports whether the profile already sets one of these flags, in // either "--flag value" or "--flag=value" form. func hasFlag(args []string, names ...string) bool { for _, a := range args { for _, n := range names { if a == n || strings.HasPrefix(a, n+"=") { return true } } } return false }