Place llama.cpp across several GPUs of a node with gpus_per_node

tp on a llama.cpp profile was a width nothing placed. gpus_per_node is
now the count: placement reserves that many cards on one node and the
engine splits layers across them. tp, pp and nodes are refused with
what to write instead.

Refs #2 (item 8).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 11:56:34 +02:00
1 parent 057be16a92
commit cbd5521f4f
5 files changed
+99 -13

No files matched your search

+20 -13
View File
@@ -67,25 +67,25 @@ func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
// Validate checks the profile against the GPUs it would run on.
//
// The two refusals here are about not pretending: llama.cpp has no equivalent
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
// different mechanism (--split-mode, --tensor-split) that this adapter does not
// drive. Accepting either would produce a replica that starts and then serves
// from one machine while the catalog believes it has several.
// llama.cpp spreads a model over a node's GPUs by layer (--split-mode layer),
// so gpus_per_node is how many it gets. It has no tensor parallelism and no
// group spanning machines, and accepting tp or nodes would produce a replica
// that serves from fewer cards, or one machine, while the catalog believes
// otherwise.
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
if err := validateCommon(p, gpus); err != nil {
return err
}
if p.Nodes > 1 {
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
p.Name, p.Nodes)
}
if p.TP > 1 {
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
p.Name)
if p.TP > 1 || p.PP > 1 {
return fmt.Errorf("profile %q: llamacpp splits a model across a node's GPUs by layer, not by tp or pp; "+
"write gpus_per_node: %d instead", p.Name, max(p.TP, 1)*max(p.PP, 1))
}
return nil
// The common checks count a replica's GPUs as tp; for llama.cpp the
// count is gpus_per_node.
q := p
q.TP = max(p.GPUsPerNode, 1)
return validateCommon(q, gpus)
}
// Commands returns the processes for one shard, which is always one process.
@@ -134,6 +134,13 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
args = append(args, "--n-gpu-layers", "999")
}
// Several GPUs: the layers spread across all of them, which is the only
// split llama.cpp does without a matched interconnect. In proportion to
// each card's free memory, which for the whole cards placement gives is
// an even split.
if len(s.GPUIndices) > 1 && !hasFlag(p.Args, "--split-mode", "-sm") {
args = append(args, "--split-mode", "layer")
}
if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
// What placement reserved first: it is what the node's other
// replicas were placed around.
+27
View File
@@ -334,3 +334,30 @@ func TestDevicesAreReadFromListDevices(t *testing.T) {
t.Errorf("no devices: %q", got)
}
}
func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"}
if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil {
t.Fatalf("four GPUs on a four-GPU node: %v", err)
}
if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil {
t.Error("four GPUs were accepted on a node with two")
}
s := llamaSpec([]int{0, 1, 2, 3})
s.Profile.GPUsPerNode = 4
cmds, err := (&LlamaCpp{}).Commands(s)
if err != nil {
t.Fatal(err)
}
if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" {
t.Errorf("--split-mode %q over four GPUs", v)
}
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") {
t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env)
}
one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
if hasFlag(one[0].Args, "--split-mode") {
t.Error("a one-GPU replica was given a split mode")
}
}