Add the llama.cpp engine, and let a profile ask for no GPU

llama.cpp is what makes a cluster with no cards useful: it serves the same
OpenAI API on CPU, so the whole path — placement, supervision, scraping,
scheduling, admission — can be exercised end to end on ordinary machines.

Three things had to give way for it.

A profile may now omit its GPU class, and that is how it says it runs on the
CPU. Only engines that can actually serve without a card may omit it; for
vLLM an omitted class stays an error, because otherwise a typo becomes a
replica placed on a machine with no hardware and diagnosed minutes later
during model loading. Placement spreads CPU replicas instead of bin packing
them, since unlike GPUs there is no allocation to stop two of them sharing a
machine's cores.

Metric mappings are now per engine rather than per cluster. llama.cpp calls
the same three numbers requests_processing, requests_deferred and
kv_cache_usage_ratio, and a single global mapping would leave half a mixed
fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that
names its own Running metric is taken as a complete statement, so the fields
it leaves blank stay blank rather than being filled with names that engine
cannot have.

The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits
across local GPUs by a different mechanism than this adapter drives, and
accepting either would produce a replica that starts and then serves from one
machine while the catalog believes it has several.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 08:14:05 +02:00
1 parent ec1b6b04df
commit 501db3d3e8
15 files changed
+697 -22

No files matched your search

+4 -2
View File
@@ -141,6 +141,7 @@ func NewRegistry(cfg Config) *Registry {
r := &Registry{engines: map[string]Engine{}}
r.Add(&VLLM{cfg: cfg.VLLM})
r.Add(&SGLang{cfg: cfg.SGLang})
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
return r
}
@@ -168,8 +169,9 @@ func (r *Registry) Names() []string {
// Config configures the engine adapters.
type Config struct {
VLLM VLLMConfig `koanf:"vllm"`
SGLang SGLangConfig `koanf:"sglang"`
VLLM VLLMConfig `koanf:"vllm"`
SGLang SGLangConfig `koanf:"sglang"`
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
}
// --- shared helpers ---
+2 -2
View File
@@ -397,14 +397,14 @@ func TestValidateRequiresModelPath(t *testing.T) {
func TestRegistry(t *testing.T) {
r := NewRegistry(Config{})
if got := r.Names(); len(got) != 2 || got[0] != "sglang" || got[1] != "vllm" {
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "vllm"}) {
t.Errorf("registry contents: %v", got)
}
if _, err := r.Get("vllm"); err != nil {
t.Errorf("vllm missing: %v", err)
}
err := func() error { _, err := r.Get("nope"); return err }()
if err == nil || !strings.Contains(err.Error(), "sglang, vllm") {
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, vllm") {
t.Errorf("an unknown engine should list what is available: %v", err)
}
}
+117
View File
@@ -0,0 +1,117 @@
package engines
import (
"fmt"
"strconv"
"strings"
)
// LlamaCppConfig locates the llama.cpp installation.
type LlamaCppConfig struct {
// Bin is the llama-server binary. It is the server, not the CLI: the
// gateway talks OpenAI over HTTP and needs the one that serves it.
Bin string `koanf:"bin"`
// Parallel is how many requests a server handles at once, when a profile
// does not say. llama.cpp defaults to one, which would make every second
// request wait behind the first no matter what the router decided.
Parallel int `koanf:"parallel"`
}
// LlamaCpp builds llama.cpp launch commands.
//
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
// small model on CPU at a speed that is fine for testing the whole path, and it
// serves the same OpenAI API as the others, so nothing downstream can tell the
// difference.
type LlamaCpp struct{ cfg LlamaCppConfig }
// Name identifies the engine in the catalog.
func (*LlamaCpp) Name() string { return "llamacpp" }
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
// is still loading, which is exactly what the supervisor needs.
func (*LlamaCpp) HealthPath() string { return "/health" }
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
// server was started with --metrics. Commands always passes it.
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
// does, because a llama.cpp replica is always exactly one shard.
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
// Validate checks the profile against the GPUs it would run on.
//
// The two refusals here are about not pretending: llama.cpp has no equivalent
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
// different mechanism (--split-mode, --tensor-split) that this adapter does not
// drive. Accepting either would produce a replica that starts and then serves
// from one machine while the catalog believes it has several.
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
if err := validateCommon(p, gpus); err != nil {
return err
}
if p.Nodes > 1 {
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
p.Name, p.Nodes)
}
if p.TP > 1 {
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
p.Name)
}
return nil
}
// Commands returns the processes for one shard, which is always one process.
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
bin := l.cfg.Bin
if bin == "" {
bin = "llama-server"
}
p := s.Profile
args := []string{
"--model", p.ModelPath,
"--host", "0.0.0.0",
"--port", strconv.Itoa(s.HTTPPort),
// Without this there is no /metrics, and an endpoint the router cannot
// measure is one it has to route to blind.
"--metrics",
}
if p.ServedModelName != "" {
args = append(args, "--alias", p.ServedModelName)
}
if !hasFlag(p.Args, "--parallel", "-np") {
n := l.cfg.Parallel
if n <= 0 {
n = 4
}
args = append(args, "--parallel", strconv.Itoa(n))
}
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
// CPU otherwise, which is not what a GPU placement asked for. Without them
// the flag is omitted entirely rather than set to zero, so a build with no
// GPU support does not see an argument it rejects.
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
args = append(args, "--n-gpu-layers", "999")
}
args = append(args, p.Args...)
return []Command{{
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
}}, nil
}
// hasFlag reports whether the profile already sets one of these flags, in
// either "--flag value" or "--flag=value" form.
func hasFlag(args []string, names ...string) bool {
for _, a := range args {
for _, n := range names {
if a == n || strings.HasPrefix(a, n+"=") {
return true
}
}
}
return false
}
+156
View File
@@ -0,0 +1,156 @@
package engines
import (
"slices"
"strings"
"testing"
)
func llamaSpec(gpus []int) ShardSpec {
return ShardSpec{
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
Profile: Profile{
Name: "cpu", Engine: "llamacpp", TP: 1,
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
},
}
}
func TestLlamaCppCommand(t *testing.T) {
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
}
c := cmds[0]
if c.Path != "llama-server" {
t.Errorf("path %q", c.Path)
}
// --metrics is not optional: llama-server serves no /metrics without it, and
// an endpoint the router cannot measure is one it routes to blind.
if !slices.Contains(c.Args, "--metrics") {
t.Error("the server was started without --metrics")
}
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
t.Errorf("--model %q", v)
}
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
t.Errorf("--port %q", v)
}
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
t.Errorf("--alias %q", v)
}
// llama.cpp serves one request at a time unless told otherwise, which would
// make every second request wait no matter what the router decided.
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
t.Errorf("--parallel %q", v)
}
if slices.Contains(c.Args, "--n-gpu-layers") {
t.Error("a CPU placement was given GPU layers")
}
}
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
t.Errorf("a GPU placement should offload the model, got %q", v)
}
// The shard must still be pinned to its own card.
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
t.Error("the shard was not pinned to its GPU")
}
}
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
// The escape hatch has to actually work, or an operator tuning slot count
// ends up with two --parallel flags and whichever llama.cpp happens to
// prefer.
s := llamaSpec(nil)
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
cmds, _ := (&LlamaCpp{}).Commands(s)
n := 0
for _, a := range cmds[0].Args {
if a == "--parallel" {
n++
}
}
if n != 1 {
t.Errorf("--parallel appears %d times", n)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
t.Errorf("the profile's value lost: %q", v)
}
}
func TestShortFormArgumentsAreRecognized(t *testing.T) {
s := llamaSpec([]int{0})
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
cmds, _ := (&LlamaCpp{}).Commands(s)
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
if slices.Contains(cmds[0].Args, flag) {
t.Errorf("%s was added although the profile set its short form", flag)
}
}
}
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
// Accepting either of these would produce a replica that starts and then
// serves from one machine while the catalog believes it has several.
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
t.Error("a multi-node llama.cpp profile was accepted")
}
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
t.Error("a tensor-parallel llama.cpp profile was accepted")
}
}
func TestCPUProfileValidates(t *testing.T) {
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
}
}
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
// Unlike the others there is no worker rank to get this wrong for.
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
t.Error("a llama.cpp shard must answer the API")
}
}
func TestBinaryIsConfigurable(t *testing.T) {
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
cmds, _ := l.Commands(llamaSpec(nil))
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
t.Errorf("path %q", cmds[0].Path)
}
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
t.Errorf("--parallel %q", v)
}
}
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
_, err := NewRegistry(Config{}).Get("llama.cpp")
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
t.Errorf("the suggestion should name the real engine: %v", err)
}
}
// flagValue reads a "--flag value" pair out of a command line.
func flagValue(args []string, flag string) (string, bool) {
for i, a := range args {
if a == flag && i+1 < len(args) {
return args[i+1], true
}
}
return "", false
}
+9 -6
View File
@@ -143,12 +143,15 @@ type Scoring struct {
// Endpoint is one engine replica. Static in phase 1.
type Endpoint struct {
ID string `koanf:"id"`
URL string `koanf:"url"`
Models []string `koanf:"models"`
Pool string `koanf:"pool"`
Weight float64 `koanf:"weight"`
Priority int `koanf:"priority"`
ID string `koanf:"id"`
URL string `koanf:"url"`
Models []string `koanf:"models"`
// Engine names the inference engine, which decides what its metrics are
// called. Empty means vLLM's names.
Engine string `koanf:"engine"`
Pool string `koanf:"pool"`
Weight float64 `koanf:"weight"`
Priority int `koanf:"priority"`
}
// Default returns the configuration used when nothing is specified.
+29 -3
View File
@@ -51,6 +51,20 @@ type Profile struct {
StartupTimeout time.Duration `koanf:"startup_timeout"`
}
// CPUOnly reports whether this profile needs no GPUs.
//
// Naming no GPU class is how a profile says so, and only engines that can
// actually serve without a card are allowed to leave it out — otherwise an
// omitted class would quietly become a replica placed on a machine with no
// hardware to run it.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
// cpuCapable reports whether an engine can serve a model without a GPU.
//
// llama.cpp is the one that can, and it is what makes a cluster with no cards
// testable end to end.
func cpuCapable(engine string) bool { return engine == "llamacpp" }
// Model is a catalog entry as clients see it.
type Model struct {
Name string `koanf:"name"`
@@ -136,7 +150,11 @@ func (m Model) normalize() Model {
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
switch {
case p.CPUOnly():
// Nothing to divide up: this profile asks for no cards at all.
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
// The common case: a single-node profile uses TP GPUs on one node.
p.GPUsPerNode = p.TP / p.Nodes
if p.GPUsPerNode <= 0 {
@@ -177,8 +195,16 @@ func (m Model) validate() error {
}
seen[p.Name] = struct{}{}
if p.GPUClass == "" {
return fmt.Errorf("model %q profile %q: gpu_class is required", m.Name, p.Name)
if p.GPUClass == "" && !cpuCapable(p.Engine) {
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
m.Name, p.Name, p.Engine)
}
if p.CPUOnly() {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
m.Name, p.Name, p.Nodes)
}
continue
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
+77
View File
@@ -0,0 +1,77 @@
package catalog
import (
"strings"
"testing"
)
func TestCPUProfileMayOmitGPUClass(t *testing.T) {
c, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp"}},
}})
if err != nil {
t.Fatalf("a llama.cpp profile with no GPU class was rejected: %v", err)
}
m, _ := c.Resolve("qwen-0.5b")
p := m.Profiles[0]
if !p.CPUOnly() {
t.Error("the profile was not recognized as CPU-only")
}
if p.GPUsPerNode != 0 {
t.Errorf("a CPU profile asks for %d GPUs per node", p.GPUsPerNode)
}
if pp := m.PlacementProfiles()[0]; !pp.CPUOnly() || pp.GPUsPerNode != 0 {
t.Errorf("placement sees %+v", pp)
}
}
func TestGPUEngineStillRequiresItsClass(t *testing.T) {
// Omitting the class for vLLM is a mistake, and one that would otherwise
// surface as a replica placed on a machine with no cards, minutes later.
_, err := New([]Model{{
Name: "llama-70b",
Source: "meta/llama",
Profiles: []Profile{{Name: "tp4", Engine: "vllm", TP: 4}},
}})
if err == nil || !strings.Contains(err.Error(), "gpu_class is required") {
t.Errorf("expected the class to be required for vllm: %v", err)
}
}
func TestCPUProfileCannotSpanNodes(t *testing.T) {
// llama.cpp has no equivalent of a replica spanning machines, so accepting
// this would place shards that never form one replica.
_, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Nodes: 2}},
}})
if err == nil || !strings.Contains(err.Error(), "cannot span") {
t.Errorf("expected a multi-node CPU profile to be refused: %v", err)
}
}
func TestCPUProfileReachesTheEngineWithoutGPUs(t *testing.T) {
c, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Args: []string{"--ctx-size", "4096"}}},
}})
if err != nil {
t.Fatal(err)
}
m, _ := c.Resolve("qwen-0.5b")
ep, ok := m.EngineProfile("cpu")
if !ok {
t.Fatal("the engine profile is missing")
}
if ep.Engine != "llamacpp" || ep.ModelPath != "/models/qwen.gguf" {
t.Errorf("engine profile %+v", ep)
}
if ep.ServedModelName != "qwen-0.5b" {
t.Errorf("served as %q", ep.ServedModelName)
}
}
+2 -1
View File
@@ -164,6 +164,7 @@ func (c *Controller) endpoints() []endpoint.Info {
ID: rep.ID,
URL: strings.TrimRight(rep.Endpoint, "/"),
Models: names,
Engine: prof.Engine,
Pool: rep.Pool,
Weight: rep.Weight,
Priority: prof.Priority,
@@ -176,7 +177,7 @@ func (c *Controller) endpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Pool, i.Models, i.Weight, i.Priority)
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
}
return b.String()
}
+102
View File
@@ -0,0 +1,102 @@
package placement
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// cpuProfile is what the catalog produces for a llama.cpp model: no GPU class,
// because there is no card to ask for.
func cpuProfile(name string) Profile {
return Profile{Name: name, Engine: "llamacpp"}
}
func TestCPUProfilePlacesOnANodeWithNoGPUs(t *testing.T) {
// The whole point: a machine with no cards is useless to every other engine
// and perfectly fine for this one.
s := New(store(node("n1", "", 0)))
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if len(res.Placement.Nodes) != 1 || res.Placement.Nodes[0].NodeID != "n1" {
t.Fatalf("placed on %+v", res.Placement.Nodes)
}
if len(res.Placement.Nodes[0].GPUIndices) != 0 {
t.Errorf("a CPU replica was given GPUs: %v", res.Placement.Nodes[0].GPUIndices)
}
if res.Placement.Pool != "none/cpu" {
t.Errorf("pool %q", res.Placement.Pool)
}
}
func TestCPUReplicasSpreadRatherThanStack(t *testing.T) {
// There is no allocation to stop two CPU replicas sharing a machine's
// cores, so placement has to do it. Bin packing, which is right for GPUs,
// would put every replica on the same node.
st := store(node("n1", "", 0), node("n2", "", 0), node("n3", "", 0))
s := New(st)
seen := map[string]int{}
for i := range 3 {
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("iteration %d:\n%s", i, res.Explain())
}
id := res.Placement.Nodes[0].NodeID
seen[id]++
// Record it the way the reconciler would, so the next decision sees it.
st.Allocate(id, inventory.Allocation{ShardID: id + "-sh", ReplicaID: id + "-rep"})
}
if len(seen) != 3 {
t.Errorf("three replicas landed on %d nodes: %v", len(seen), seen)
}
}
func TestCPUProfileIgnoresGPUsItDoesNotNeed(t *testing.T) {
// A GPU node is still a valid host, and taking it must not consume cards
// that a GPU profile would need.
st := store(node("gpu", "h100-80g", 8))
s := New(st)
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if n := len(res.Placement.Nodes[0].GPUIndices); n != 0 {
t.Errorf("a CPU replica claimed %d GPUs", n)
}
if free := st.FreeGPUs("gpu"); len(free) != 8 {
t.Errorf("GPUs were consumed: %d free", len(free))
}
}
func TestGPUProfileStillNeedsItsClass(t *testing.T) {
// The CPU path must not become a way for a GPU profile to be placed on a
// machine that cannot run it.
s := New(store(node("n1", "", 0)))
res := s.Place([]Profile{{Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1}}, nil)
if res.Placement != nil {
t.Fatal("a GPU profile was placed on a node with no GPUs")
}
if len(res.Rejections) == 0 {
t.Error("the rejection should say why")
}
}
func TestCPUProfileRespectsAvoid(t *testing.T) {
// Spreading replicas of one model across failure domains matters the same
// way it does for GPU replicas.
s := New(store(node("n1", "", 0), node("n2", "", 0)))
res := s.Place([]Profile{cpuProfile("cpu")}, []string{"n1"})
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "n2" {
t.Errorf("placed on %q despite being told to avoid it", got)
}
}
+40 -4
View File
@@ -34,12 +34,27 @@ type Profile struct {
Priority int
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
// profile omit its GPU class when its engine can serve without one.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" }
// classLabel names the profile's hardware for pools and messages.
func (p Profile) classLabel() string {
if p.CPUOnly() {
return "cpu"
}
return p.GPUClass
}
// normalize fills in the defaults a catalog entry may omit.
func (p Profile) normalize() Profile {
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
switch {
case p.CPUOnly():
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
p.GPUsPerNode = 1
}
return p
@@ -166,6 +181,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return len(candidates[i].free) < len(candidates[j].free)
})
if p.CPUOnly() {
// Nothing to bin pack, so spread instead: two CPU replicas on one
// machine compete for the same cores, and unlike GPUs there is no
// allocation to stop them.
sort.SliceStable(candidates, func(i, j int) bool {
return len(s.inv.Allocations(candidates[i].node.ID)) <
len(s.inv.Allocations(candidates[j].node.ID))
})
return &Placement{
Profile: p,
Pool: "none/cpu",
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}},
}, rejections
}
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode)
if gpus == nil {
@@ -178,7 +208,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return &Placement{
Profile: p,
Pool: "none/" + p.GPUClass,
Pool: "none/" + p.classLabel(),
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
}, rejections
}
@@ -201,7 +231,7 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
}
for _, fabric := range fabrics {
pool := fabric + "/" + p.GPUClass
pool := fabric + "/" + p.classLabel()
candidates := s.eligibleNodes(p, fabric, avoid, &rejections)
if len(candidates) < p.Nodes {
@@ -262,7 +292,7 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
if fabric != "" {
pool = fabric
}
pool += "/" + p.GPUClass
pool += "/" + p.classLabel()
var out []candidate
for _, n := range s.inv.Nodes() {
@@ -290,6 +320,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
}
}
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
out = append(out, candidate{node: n})
continue
}
freeIdx := s.inv.FreeGPUs(n.ID)
var free []discovery.GPU
for _, g := range n.GPUs {
+3
View File
@@ -24,6 +24,9 @@ type Info struct {
URL string
// Models lists the model names this replica serves.
Models []string
// Engine names the inference engine serving here, which decides what its
// metrics are called. Empty means vLLM's names.
Engine string
// Pool names the (fabric, gpu_class) pool. Used for observability and pool
// preference only; phase 1 may well have a single pool.
Pool string
+1
View File
@@ -168,6 +168,7 @@ func toInfos(eps []config.Endpoint) []endpoint.Info {
ID: e.ID,
URL: strings.TrimRight(e.URL, "/"),
Models: e.Models,
Engine: e.Engine,
Pool: e.Pool,
Weight: e.Weight,
Priority: e.Priority,
+96
View File
@@ -0,0 +1,96 @@
package scrape
import (
"errors"
"strings"
"testing"
)
// llamaMetrics is what llama-server --metrics actually serves, trimmed to the
// families that matter.
const llamaMetrics = `# HELP llamacpp:prompt_tokens_total Number of prompt tokens processed.
# TYPE llamacpp:prompt_tokens_total counter
llamacpp:prompt_tokens_total 1200
# HELP llamacpp:tokens_predicted_total Number of generation tokens processed.
# TYPE llamacpp:tokens_predicted_total counter
llamacpp:tokens_predicted_total 3400
# HELP llamacpp:kv_cache_usage_ratio KV-cache usage.
# TYPE llamacpp:kv_cache_usage_ratio gauge
llamacpp:kv_cache_usage_ratio 0.42
# HELP llamacpp:requests_processing Number of requests processing.
# TYPE llamacpp:requests_processing gauge
llamacpp:requests_processing 3
# HELP llamacpp:requests_deferred Number of requests deferred.
# TYPE llamacpp:requests_deferred gauge
llamacpp:requests_deferred 7
`
func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) {
s, err := Parse(strings.NewReader(llamaMetrics), LlamaCppMapping())
if err != nil {
t.Fatal(err)
}
if s.Running != 3 {
t.Errorf("running=%d", s.Running)
}
// "Deferred" is llama.cpp's word for a request that arrived with every slot
// busy, which is what the scheduler needs to read as queue depth.
if s.Waiting != 7 {
t.Errorf("waiting=%d", s.Waiting)
}
if s.KVCacheUsage != 0.42 {
t.Errorf("kv=%v", s.KVCacheUsage)
}
if s.PromptTokens != 1200 || s.GenerationTokens != 3400 {
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
}
}
func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) {
// llama.cpp has no request-success counter. Filling that field with vLLM's
// name would send the router hunting for a metric that cannot be there, and
// would hide the fact that this engine simply does not report it.
m := LlamaCppMapping()
m.normalize()
if m.SuccessTotal != "" || m.CacheConfig != "" {
t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m)
}
if m.Running != "llamacpp:requests_processing" {
t.Errorf("running=%q", m.Running)
}
}
func TestEmptyMappingStillDefaultsToVLLM(t *testing.T) {
var m Mapping
m.normalize()
if m.Running != "vllm:num_requests_running" || m.SuccessTotal == "" {
t.Errorf("an unconfigured mapping should be vLLM's: %+v", m)
}
}
func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) {
// The failure this guards against is silent: reading a vLLM endpoint with
// llama.cpp's vocabulary finds nothing, and an endpoint with no numbers
// must be reported as unmeasurable rather than as idle.
var missing *MissingError
_, err := Parse(strings.NewReader(llamaMetrics), VLLMMapping())
if !errors.As(err, &missing) {
t.Fatalf("expected the contract to be reported incomplete, got %v", err)
}
if len(missing.Missing) != 3 {
t.Errorf("all three mandatory metrics should be named: %v", missing.Missing)
}
}
func TestMappingForPicksByEngine(t *testing.T) {
if MappingFor("llamacpp").Running != "llamacpp:requests_processing" {
t.Error("llamacpp did not get its own mapping")
}
// An endpoint that names no engine, or one this build has never heard of,
// is read with vLLM's names rather than left unreadable.
for _, engine := range []string{"", "vllm", "something-new"} {
if MappingFor(engine).Running != "vllm:num_requests_running" {
t.Errorf("engine %q did not fall back to vLLM", engine)
}
}
}
+43
View File
@@ -30,6 +30,40 @@ type Mapping struct {
SuccessTotal string
}
// MappingFor returns the metric names for an engine.
//
// Per engine rather than per cluster: a cluster running llama.cpp on the CPU
// nodes and vLLM on the GPU nodes has two vocabularies for the same three
// numbers, and one global mapping would leave half the fleet unmeasurable.
// An unknown engine falls back to vLLM's names, which is also what an endpoint
// that names no engine gets.
func MappingFor(engine string) Mapping {
switch engine {
case "llamacpp":
return LlamaCppMapping()
default:
return VLLMMapping()
}
}
// LlamaCppMapping returns llama.cpp's metric names, which llama-server only
// exposes when started with --metrics.
//
// "Deferred" is llama.cpp's word for a request that arrived with every slot
// busy, which is the same thing vLLM calls waiting. The fields left empty are
// metrics llama.cpp does not have: it counts decodes rather than requests, so
// there is no success counter and no cache configuration to read block sizes
// from.
func LlamaCppMapping() Mapping {
return Mapping{
Running: "llamacpp:requests_processing",
Waiting: "llamacpp:requests_deferred",
KVCacheUsage: "llamacpp:kv_cache_usage_ratio",
PromptTokens: "llamacpp:prompt_tokens_total",
GenerationTokens: "llamacpp:tokens_predicted_total",
}
}
// VLLMMapping returns vLLM's current metric names.
func VLLMMapping() Mapping {
return Mapping{
@@ -43,7 +77,16 @@ func VLLMMapping() Mapping {
}
}
// normalize fills in vLLM's names for an empty mapping.
//
// Only for an empty one. A mapping that names its own Running metric is a
// complete statement about some other engine, and the fields it leaves blank
// are metrics that engine does not expose — filling those with vLLM's names
// would have the router hunting for metrics that cannot be there.
func (m *Mapping) normalize() {
if m.Running != "" {
return
}
d := VLLMMapping()
if m.Running == "" {
m.Running = d.Running
+16 -4
View File
@@ -20,7 +20,8 @@ type Config struct {
Timeout time.Duration
// Path is the metrics endpoint.
Path string
// Mapping selects the metric names.
// Mapping selects the metric names for endpoints that name no engine. An
// endpoint that names one is read with that engine's mapping instead.
Mapping Mapping
}
@@ -152,7 +153,7 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
}
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
stats, err := p.fetch(ctx, in.URL)
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
if err != nil {
if ctx.Err() != nil {
return
@@ -164,7 +165,7 @@ func (p *Poller) once(ctx context.Context, in endpoint.Info) {
p.reg.Observe(in.ID, stats)
}
func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error) {
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()
@@ -181,5 +182,16 @@ func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error)
if resp.StatusCode != http.StatusOK {
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
}
return Parse(resp.Body, p.cfg.Mapping)
return Parse(resp.Body, mapping)
}
// mappingFor picks the vocabulary this endpoint's metrics are written in.
//
// The configured mapping wins for endpoints that name no engine, so an operator
// can still override the names when a version renames them (plan §14).
func (p *Poller) mappingFor(in endpoint.Info) Mapping {
if in.Engine == "" {
return p.cfg.Mapping
}
return MappingFor(in.Engine)
}