Add the llama.cpp engine, and let a profile ask for no GPU

llama.cpp is what makes a cluster with no cards useful: it serves the same
OpenAI API on CPU, so the whole path — placement, supervision, scraping,
scheduling, admission — can be exercised end to end on ordinary machines.

Three things had to give way for it.

A profile may now omit its GPU class, and that is how it says it runs on the
CPU. Only engines that can actually serve without a card may omit it; for
vLLM an omitted class stays an error, because otherwise a typo becomes a
replica placed on a machine with no hardware and diagnosed minutes later
during model loading. Placement spreads CPU replicas instead of bin packing
them, since unlike GPUs there is no allocation to stop two of them sharing a
machine's cores.

Metric mappings are now per engine rather than per cluster. llama.cpp calls
the same three numbers requests_processing, requests_deferred and
kv_cache_usage_ratio, and a single global mapping would leave half a mixed
fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that
names its own Running metric is taken as a complete statement, so the fields
it leaves blank stay blank rather than being filled with names that engine
cannot have.

The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits
across local GPUs by a different mechanism than this adapter drives, and
accepting either would produce a replica that starts and then serves from one
machine while the catalog believes it has several.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 08:14:05 +02:00
1 parent ec1b6b04df
commit 501db3d3e8
15 files changed
+697 -22

No files matched your search

+29 -3
View File
@@ -51,6 +51,20 @@ type Profile struct {
StartupTimeout time.Duration `koanf:"startup_timeout"`
}
// CPUOnly reports whether this profile needs no GPUs.
//
// Naming no GPU class is how a profile says so, and only engines that can
// actually serve without a card are allowed to leave it out — otherwise an
// omitted class would quietly become a replica placed on a machine with no
// hardware to run it.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) }
// cpuCapable reports whether an engine can serve a model without a GPU.
//
// llama.cpp is the one that can, and it is what makes a cluster with no cards
// testable end to end.
func cpuCapable(engine string) bool { return engine == "llamacpp" }
// Model is a catalog entry as clients see it.
type Model struct {
Name string `koanf:"name"`
@@ -136,7 +150,11 @@ func (m Model) normalize() Model {
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
switch {
case p.CPUOnly():
// Nothing to divide up: this profile asks for no cards at all.
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
// The common case: a single-node profile uses TP GPUs on one node.
p.GPUsPerNode = p.TP / p.Nodes
if p.GPUsPerNode <= 0 {
@@ -177,8 +195,16 @@ func (m Model) validate() error {
}
seen[p.Name] = struct{}{}
if p.GPUClass == "" {
return fmt.Errorf("model %q profile %q: gpu_class is required", m.Name, p.Name)
if p.GPUClass == "" && !cpuCapable(p.Engine) {
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
m.Name, p.Name, p.Engine)
}
if p.CPUOnly() {
if p.Nodes > 1 {
return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes",
m.Name, p.Name, p.Nodes)
}
continue
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
+77
View File
@@ -0,0 +1,77 @@
package catalog
import (
"strings"
"testing"
)
func TestCPUProfileMayOmitGPUClass(t *testing.T) {
c, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp"}},
}})
if err != nil {
t.Fatalf("a llama.cpp profile with no GPU class was rejected: %v", err)
}
m, _ := c.Resolve("qwen-0.5b")
p := m.Profiles[0]
if !p.CPUOnly() {
t.Error("the profile was not recognized as CPU-only")
}
if p.GPUsPerNode != 0 {
t.Errorf("a CPU profile asks for %d GPUs per node", p.GPUsPerNode)
}
if pp := m.PlacementProfiles()[0]; !pp.CPUOnly() || pp.GPUsPerNode != 0 {
t.Errorf("placement sees %+v", pp)
}
}
func TestGPUEngineStillRequiresItsClass(t *testing.T) {
// Omitting the class for vLLM is a mistake, and one that would otherwise
// surface as a replica placed on a machine with no cards, minutes later.
_, err := New([]Model{{
Name: "llama-70b",
Source: "meta/llama",
Profiles: []Profile{{Name: "tp4", Engine: "vllm", TP: 4}},
}})
if err == nil || !strings.Contains(err.Error(), "gpu_class is required") {
t.Errorf("expected the class to be required for vllm: %v", err)
}
}
func TestCPUProfileCannotSpanNodes(t *testing.T) {
// llama.cpp has no equivalent of a replica spanning machines, so accepting
// this would place shards that never form one replica.
_, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Nodes: 2}},
}})
if err == nil || !strings.Contains(err.Error(), "cannot span") {
t.Errorf("expected a multi-node CPU profile to be refused: %v", err)
}
}
func TestCPUProfileReachesTheEngineWithoutGPUs(t *testing.T) {
c, err := New([]Model{{
Name: "qwen-0.5b",
Source: "/models/qwen.gguf",
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Args: []string{"--ctx-size", "4096"}}},
}})
if err != nil {
t.Fatal(err)
}
m, _ := c.Resolve("qwen-0.5b")
ep, ok := m.EngineProfile("cpu")
if !ok {
t.Fatal("the engine profile is missing")
}
if ep.Engine != "llamacpp" || ep.ModelPath != "/models/qwen.gguf" {
t.Errorf("engine profile %+v", ep)
}
if ep.ServedModelName != "qwen-0.5b" {
t.Errorf("served as %q", ep.ServedModelName)
}
}
+2 -1
View File
@@ -164,6 +164,7 @@ func (c *Controller) endpoints() []endpoint.Info {
ID: rep.ID,
URL: strings.TrimRight(rep.Endpoint, "/"),
Models: names,
Engine: prof.Engine,
Pool: rep.Pool,
Weight: rep.Weight,
Priority: prof.Priority,
@@ -176,7 +177,7 @@ func (c *Controller) endpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Pool, i.Models, i.Weight, i.Priority)
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
}
return b.String()
}
+102
View File
@@ -0,0 +1,102 @@
package placement
import (
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// cpuProfile is what the catalog produces for a llama.cpp model: no GPU class,
// because there is no card to ask for.
func cpuProfile(name string) Profile {
return Profile{Name: name, Engine: "llamacpp"}
}
func TestCPUProfilePlacesOnANodeWithNoGPUs(t *testing.T) {
// The whole point: a machine with no cards is useless to every other engine
// and perfectly fine for this one.
s := New(store(node("n1", "", 0)))
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if len(res.Placement.Nodes) != 1 || res.Placement.Nodes[0].NodeID != "n1" {
t.Fatalf("placed on %+v", res.Placement.Nodes)
}
if len(res.Placement.Nodes[0].GPUIndices) != 0 {
t.Errorf("a CPU replica was given GPUs: %v", res.Placement.Nodes[0].GPUIndices)
}
if res.Placement.Pool != "none/cpu" {
t.Errorf("pool %q", res.Placement.Pool)
}
}
func TestCPUReplicasSpreadRatherThanStack(t *testing.T) {
// There is no allocation to stop two CPU replicas sharing a machine's
// cores, so placement has to do it. Bin packing, which is right for GPUs,
// would put every replica on the same node.
st := store(node("n1", "", 0), node("n2", "", 0), node("n3", "", 0))
s := New(st)
seen := map[string]int{}
for i := range 3 {
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("iteration %d:\n%s", i, res.Explain())
}
id := res.Placement.Nodes[0].NodeID
seen[id]++
// Record it the way the reconciler would, so the next decision sees it.
st.Allocate(id, inventory.Allocation{ShardID: id + "-sh", ReplicaID: id + "-rep"})
}
if len(seen) != 3 {
t.Errorf("three replicas landed on %d nodes: %v", len(seen), seen)
}
}
func TestCPUProfileIgnoresGPUsItDoesNotNeed(t *testing.T) {
// A GPU node is still a valid host, and taking it must not consume cards
// that a GPU profile would need.
st := store(node("gpu", "h100-80g", 8))
s := New(st)
res := s.Place([]Profile{cpuProfile("cpu")}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if n := len(res.Placement.Nodes[0].GPUIndices); n != 0 {
t.Errorf("a CPU replica claimed %d GPUs", n)
}
if free := st.FreeGPUs("gpu"); len(free) != 8 {
t.Errorf("GPUs were consumed: %d free", len(free))
}
}
func TestGPUProfileStillNeedsItsClass(t *testing.T) {
// The CPU path must not become a way for a GPU profile to be placed on a
// machine that cannot run it.
s := New(store(node("n1", "", 0)))
res := s.Place([]Profile{{Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1}}, nil)
if res.Placement != nil {
t.Fatal("a GPU profile was placed on a node with no GPUs")
}
if len(res.Rejections) == 0 {
t.Error("the rejection should say why")
}
}
func TestCPUProfileRespectsAvoid(t *testing.T) {
// Spreading replicas of one model across failure domains matters the same
// way it does for GPU replicas.
s := New(store(node("n1", "", 0), node("n2", "", 0)))
res := s.Place([]Profile{cpuProfile("cpu")}, []string{"n1"})
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "n2" {
t.Errorf("placed on %q despite being told to avoid it", got)
}
}
+40 -4
View File
@@ -34,12 +34,27 @@ type Profile struct {
Priority int
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
// profile omit its GPU class when its engine can serve without one.
func (p Profile) CPUOnly() bool { return p.GPUClass == "" }
// classLabel names the profile's hardware for pools and messages.
func (p Profile) classLabel() string {
if p.CPUOnly() {
return "cpu"
}
return p.GPUClass
}
// normalize fills in the defaults a catalog entry may omit.
func (p Profile) normalize() Profile {
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
switch {
case p.CPUOnly():
p.GPUsPerNode = 0
case p.GPUsPerNode <= 0:
p.GPUsPerNode = 1
}
return p
@@ -166,6 +181,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return len(candidates[i].free) < len(candidates[j].free)
})
if p.CPUOnly() {
// Nothing to bin pack, so spread instead: two CPU replicas on one
// machine compete for the same cores, and unlike GPUs there is no
// allocation to stop them.
sort.SliceStable(candidates, func(i, j int) bool {
return len(s.inv.Allocations(candidates[i].node.ID)) <
len(s.inv.Allocations(candidates[j].node.ID))
})
return &Placement{
Profile: p,
Pool: "none/cpu",
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}},
}, rejections
}
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode)
if gpus == nil {
@@ -178,7 +208,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return &Placement{
Profile: p,
Pool: "none/" + p.GPUClass,
Pool: "none/" + p.classLabel(),
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
}, rejections
}
@@ -201,7 +231,7 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
}
for _, fabric := range fabrics {
pool := fabric + "/" + p.GPUClass
pool := fabric + "/" + p.classLabel()
candidates := s.eligibleNodes(p, fabric, avoid, &rejections)
if len(candidates) < p.Nodes {
@@ -262,7 +292,7 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
if fabric != "" {
pool = fabric
}
pool += "/" + p.GPUClass
pool += "/" + p.classLabel()
var out []candidate
for _, n := range s.inv.Nodes() {
@@ -290,6 +320,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
}
}
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
out = append(out, candidate{node: n})
continue
}
freeIdx := s.inv.FreeGPUs(n.ID)
var free []discovery.GPU
for _, g := range n.GPUs {