Place llama.cpp across several GPUs of a node with gpus_per_node

tp on a llama.cpp profile was a width nothing placed. gpus_per_node is
now the count: placement reserves that many cards on one node and the
engine splits layers across them. tp, pp and nodes are refused with
what to write instead.

Refs #2 (item 8).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 11:56:34 +02:00
1 parent 057be16a92
commit cbd5521f4f
5 files changed
+99 -13

No files matched your search

+15
View File
@@ -381,6 +381,21 @@ func (m Model) validate() error {
}
continue
}
// llama.cpp on GPUs splits a model across one node's cards by layer,
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
// parallelism, and a tp it was given was a width nothing placed — the
// replica stayed unplaced, or ran on one card while four were meant.
if p.Engine == "llamacpp" {
switch {
case p.Nodes > 1:
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
m.Name, p.Name, p.Nodes)
case p.TP > 1 || p.PP > 1:
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+
"write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1))
}
continue
}
// The arithmetic has to add up, or the engine is handed a shard spec it
// cannot satisfy and fails minutes later during model loading.
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
+26
View File
@@ -321,3 +321,29 @@ func TestAProfileCannotSetWhatRunsCodeInTheEngine(t *testing.T) {
t.Errorf("ordinary tuning refused: %v", err)
}
}
func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
four := model("big", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", GPUsPerNode: 4})
c, err := New([]Model{four})
if err != nil {
t.Fatalf("gpus_per_node 4 on llama.cpp: %v", err)
}
m, _ := c.Resolve("big")
if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 {
t.Errorf("placement was not asked for four GPUs: %+v", pl)
}
// tp is a width llama.cpp has no way to use; refused, saying what to
// write instead.
for _, p := range []Profile{
{Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4},
{Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1},
} {
_, err := New([]Model{model("x", p)})
if err == nil {
t.Errorf("%s: accepted", p.Name)
}
if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) {
t.Errorf("tp 4 should point at gpus_per_node: %v", err)
}
}
}