Engines listen on every address their node has, with no key. Checked on the live cluster: from a neighbouring node, a request straight to an engine's port was answered — past the gateway's authentication, quotas and accounting, by anyone who could reach the node. Each replica's engine is now started with a key and refuses requests without it. The key is derived from the replica id with an HMAC over a secret kept in the store, rather than stored per replica: the router has to send the right key to replicas this process never started — adopted after a restart, or placed by the other control plane — and a function of a shared secret gives that without a table to keep in step. A key taken from one node opens that replica only. The secret is made on first use and copied with the rest of the store when moving to PostgreSQL; a new one would lock the router out of every running engine at the moment of the move. The key reaches llama.cpp and vLLM through their environment (LLAMA_API_KEY, VLLM_API_KEY) rather than their command line, which every user on the node can read; SGLang only takes it as an argument. The agent's readiness check, the router's proxy and its metrics scrape send it, the client's own Authorization still never reaches the engine, and `ctl bench` fetches the key of the replica it is pointed at from the admin API. Engines started before this keep running without a key until their replica is next started. With no store there is nowhere to keep the secret, and engines start without one, with a warning. Also formats two files that went in unformatted. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
202 lines
7.1 KiB
Go
202 lines
7.1 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// LlamaCppConfig locates the llama.cpp installation.
|
|
type LlamaCppConfig struct {
|
|
// Bin is the llama-server binary. It is the server, not the CLI: the
|
|
// gateway talks OpenAI over HTTP and needs the one that serves it.
|
|
// Empty means the agent installs llama.cpp itself, at the version the
|
|
// control plane asks for. Setting it opts out of that.
|
|
Bin string `koanf:"bin"`
|
|
// Parallel is how many requests a server handles at once, when a profile
|
|
// does not say. llama.cpp defaults to one, which would make every second
|
|
// request wait behind the first no matter what the router decided.
|
|
Parallel int `koanf:"parallel"`
|
|
}
|
|
|
|
// LlamaCpp builds llama.cpp launch commands.
|
|
//
|
|
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
|
// small model on CPU at a speed that is fine for testing the whole path, and it
|
|
// serves the same OpenAI API as the others, so nothing downstream can tell the
|
|
// difference.
|
|
type LlamaCpp struct {
|
|
cfg LlamaCppConfig
|
|
installed atomic.Pointer[string]
|
|
}
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*LlamaCpp) Name() string { return "llamacpp" }
|
|
|
|
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
|
|
// is still loading, which is exactly what the supervisor needs.
|
|
func (*LlamaCpp) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
|
|
// server was started with --metrics. Commands always passes it.
|
|
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
|
|
|
|
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
|
|
// does, because a llama.cpp replica is always exactly one shard.
|
|
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
//
|
|
// The two refusals here are about not pretending: llama.cpp has no equivalent
|
|
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
|
|
// different mechanism (--split-mode, --tensor-split) that this adapter does not
|
|
// drive. Accepting either would produce a replica that starts and then serves
|
|
// from one machine while the catalog believes it has several.
|
|
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
if p.Nodes > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
|
|
p.Name, p.Nodes)
|
|
}
|
|
if p.TP > 1 {
|
|
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
|
|
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
|
|
p.Name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Commands returns the processes for one shard, which is always one process.
|
|
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
|
bin := l.bin()
|
|
p := s.Profile
|
|
|
|
model, err := ggufIn(p.ModelPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
args := []string{
|
|
"--model", model,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
// Without this there is no /metrics, and an endpoint the router cannot
|
|
// measure is one it has to route to blind.
|
|
"--metrics",
|
|
}
|
|
// The model's own chat template rather than a generic one, which is what
|
|
// renders tool definitions in the form the model was trained to read.
|
|
//
|
|
// Pinned rather than relied on: build 10991 has it on by default, older
|
|
// builds do not, and an engine upgrade should not be able to change how
|
|
// prompts are built underneath a running fleet. Passing it explicitly was
|
|
// a no-op on the build here — it was added while chasing tool calls that
|
|
// came back as text, and did not fix that, which turned out to be the
|
|
// model rather than the template.
|
|
if !hasFlag(p.Args, "--jinja", "--no-jinja") {
|
|
args = append(args, "--jinja")
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--alias", p.ServedModelName)
|
|
}
|
|
if !hasFlag(p.Args, "--parallel", "-np") {
|
|
n := l.cfg.Parallel
|
|
if n <= 0 {
|
|
n = 4
|
|
}
|
|
args = append(args, "--parallel", strconv.Itoa(n))
|
|
}
|
|
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
|
|
// CPU otherwise, which is not what a GPU placement asked for. Without them
|
|
// the flag is omitted entirely rather than set to zero, so a build with no
|
|
// GPU support does not see an argument it rejects.
|
|
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
|
args = append(args, "--n-gpu-layers", "999")
|
|
}
|
|
args = append(args, p.Args...)
|
|
|
|
env := baseEnv(s)
|
|
if s.APIKey != "" {
|
|
env["LLAMA_API_KEY"] = s.APIKey
|
|
}
|
|
return []Command{{
|
|
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
|
|
}}, nil
|
|
}
|
|
|
|
// hasFlag reports whether the profile already sets one of these flags, in
|
|
// either "--flag value" or "--flag=value" form.
|
|
func hasFlag(args []string, names ...string) bool {
|
|
for _, a := range args {
|
|
for _, n := range names {
|
|
if a == n || strings.HasPrefix(a, n+"=") {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ggufIn resolves a model path that may be a directory.
|
|
//
|
|
// A multi-file version arrives as the directory it was downloaded into, because
|
|
// that is the only shape a set can take. llama.cpp wants a file: given the
|
|
// first part of a split GGUF it finds the rest beside it, so the first part is
|
|
// what it gets.
|
|
//
|
|
// A path that is already a file is returned untouched, which is every
|
|
// single-file model and every hand-staged one.
|
|
func ggufIn(path string) (string, error) {
|
|
st, err := os.Stat(path)
|
|
if err != nil || !st.IsDir() {
|
|
// Not a directory, or not there at all — the second is the engine's to
|
|
// report, with the message it would have given anyway.
|
|
return path, nil //nolint:nilerr // a missing path is the engine's error to give
|
|
}
|
|
entries, err := os.ReadDir(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var names []string
|
|
for _, e := range entries {
|
|
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") {
|
|
names = append(names, e.Name())
|
|
}
|
|
}
|
|
if len(names) == 0 {
|
|
return "", fmt.Errorf("%s holds no .gguf file", path)
|
|
}
|
|
// Sorted, so a split model's first part comes first: the parts are named
|
|
// "-00001-of-00004" and lexical order is shard order.
|
|
sort.Strings(names)
|
|
return filepath.Join(path, names[0]), nil
|
|
}
|
|
|
|
// UseInstalled points the engine at a build the agent installed and checked.
|
|
// It takes precedence over everything else, because it is the only one that
|
|
// was seen to run.
|
|
func (l *LlamaCpp) UseInstalled(path string) { l.installed.Store(&path) }
|
|
|
|
// Configured reports whether the agent's configuration names a binary. A
|
|
// named binary is the operator's deliberate choice; one merely found on PATH
|
|
// is not, and is not reported as installed.
|
|
func (l *LlamaCpp) Configured() bool { return l.cfg.Bin != "" }
|
|
|
|
// bin is the llama-server to run: the installed build, else the configured
|
|
// one, else whatever PATH has — which is only ever reached by a node not
|
|
// provisioned by the agent and not told where its engine is.
|
|
func (l *LlamaCpp) bin() string {
|
|
if p := l.installed.Load(); p != nil {
|
|
return *p
|
|
}
|
|
if l.cfg.Bin != "" {
|
|
return l.cfg.Bin
|
|
}
|
|
return "llama-server"
|
|
}
|