Place llama.cpp across several GPUs of a node with gpus_per_node
tp on a llama.cpp profile was a width nothing placed. gpus_per_node is now the count: placement reserves that many cards on one node and the engine splits layers across them. tp, pp and nodes are refused with what to write instead. Refs #2 (item 8). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
057be16a92
commit
cbd5521f4f
5 files changed
+99
-13
No files matched your search
@@ -47,6 +47,17 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
|
||||
node with two kinds of GPU, a shard could get a different card from the one it
|
||||
was given.
|
||||
|
||||
### llama.cpp on several GPUs of one node
|
||||
|
||||
A llama.cpp profile takes several of a node's GPUs with `gpus_per_node: N`:
|
||||
placement reserves N cards of its `gpu_class` on one node, the engine sees
|
||||
those N (`CUDA_VISIBLE_DEVICES`) and spreads the layers over them
|
||||
(`--split-mode layer`, in proportion to each card's free memory, which for
|
||||
whole cards is even). `tp` or `pp` on a llama.cpp profile was a width nothing
|
||||
placed — the replica stayed unplaced or ran on one card — and is now refused
|
||||
when the model is added, saying to write `gpus_per_node` instead; so is
|
||||
`nodes` above one.
|
||||
|
||||
## v0.10.6 — 2026-10-01
|
||||
|
||||
Published first as release candidates v0.10.6-rc.1 to rc.8; rc.5 failed its
|
||||
|
||||
@@ -67,25 +67,25 @@ func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
|
||||
|
||||
// Validate checks the profile against the GPUs it would run on.
|
||||
//
|
||||
// The two refusals here are about not pretending: llama.cpp has no equivalent
|
||||
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
|
||||
// different mechanism (--split-mode, --tensor-split) that this adapter does not
|
||||
// drive. Accepting either would produce a replica that starts and then serves
|
||||
// from one machine while the catalog believes it has several.
|
||||
// llama.cpp spreads a model over a node's GPUs by layer (--split-mode layer),
|
||||
// so gpus_per_node is how many it gets. It has no tensor parallelism and no
|
||||
// group spanning machines, and accepting tp or nodes would produce a replica
|
||||
// that serves from fewer cards, or one machine, while the catalog believes
|
||||
// otherwise.
|
||||
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
||||
if err := validateCommon(p, gpus); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.Nodes > 1 {
|
||||
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
|
||||
p.Name, p.Nodes)
|
||||
}
|
||||
if p.TP > 1 {
|
||||
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
|
||||
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
|
||||
p.Name)
|
||||
if p.TP > 1 || p.PP > 1 {
|
||||
return fmt.Errorf("profile %q: llamacpp splits a model across a node's GPUs by layer, not by tp or pp; "+
|
||||
"write gpus_per_node: %d instead", p.Name, max(p.TP, 1)*max(p.PP, 1))
|
||||
}
|
||||
return nil
|
||||
// The common checks count a replica's GPUs as tp; for llama.cpp the
|
||||
// count is gpus_per_node.
|
||||
q := p
|
||||
q.TP = max(p.GPUsPerNode, 1)
|
||||
return validateCommon(q, gpus)
|
||||
}
|
||||
|
||||
// Commands returns the processes for one shard, which is always one process.
|
||||
@@ -134,6 +134,13 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
||||
args = append(args, "--n-gpu-layers", "999")
|
||||
}
|
||||
// Several GPUs: the layers spread across all of them, which is the only
|
||||
// split llama.cpp does without a matched interconnect. In proportion to
|
||||
// each card's free memory, which for the whole cards placement gives is
|
||||
// an even split.
|
||||
if len(s.GPUIndices) > 1 && !hasFlag(p.Args, "--split-mode", "-sm") {
|
||||
args = append(args, "--split-mode", "layer")
|
||||
}
|
||||
if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
|
||||
// What placement reserved first: it is what the node's other
|
||||
// replicas were placed around.
|
||||
|
||||
@@ -334,3 +334,30 @@ func TestDevicesAreReadFromListDevices(t *testing.T) {
|
||||
t.Errorf("no devices: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
|
||||
four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"}
|
||||
if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil {
|
||||
t.Fatalf("four GPUs on a four-GPU node: %v", err)
|
||||
}
|
||||
if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil {
|
||||
t.Error("four GPUs were accepted on a node with two")
|
||||
}
|
||||
|
||||
s := llamaSpec([]int{0, 1, 2, 3})
|
||||
s.Profile.GPUsPerNode = 4
|
||||
cmds, err := (&LlamaCpp{}).Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" {
|
||||
t.Errorf("--split-mode %q over four GPUs", v)
|
||||
}
|
||||
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") {
|
||||
t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env)
|
||||
}
|
||||
one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
||||
if hasFlag(one[0].Args, "--split-mode") {
|
||||
t.Error("a one-GPU replica was given a split mode")
|
||||
}
|
||||
}
|
||||
@@ -381,6 +381,21 @@ func (m Model) validate() error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
// llama.cpp on GPUs splits a model across one node's cards by layer,
|
||||
// and how many is gpus_per_node. Not tp: llama.cpp has no tensor
|
||||
// parallelism, and a tp it was given was a width nothing placed — the
|
||||
// replica stayed unplaced, or ran on one card while four were meant.
|
||||
if p.Engine == "llamacpp" {
|
||||
switch {
|
||||
case p.Nodes > 1:
|
||||
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
|
||||
m.Name, p.Name, p.Nodes)
|
||||
case p.TP > 1 || p.PP > 1:
|
||||
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+
|
||||
"write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The arithmetic has to add up, or the engine is handed a shard spec it
|
||||
// cannot satisfy and fails minutes later during model loading.
|
||||
if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) {
|
||||
|
||||
@@ -321,3 +321,29 @@ func TestAProfileCannotSetWhatRunsCodeInTheEngine(t *testing.T) {
|
||||
t.Errorf("ordinary tuning refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
|
||||
four := model("big", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", GPUsPerNode: 4})
|
||||
c, err := New([]Model{four})
|
||||
if err != nil {
|
||||
t.Fatalf("gpus_per_node 4 on llama.cpp: %v", err)
|
||||
}
|
||||
m, _ := c.Resolve("big")
|
||||
if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 {
|
||||
t.Errorf("placement was not asked for four GPUs: %+v", pl)
|
||||
}
|
||||
// tp is a width llama.cpp has no way to use; refused, saying what to
|
||||
// write instead.
|
||||
for _, p := range []Profile{
|
||||
{Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4},
|
||||
{Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1},
|
||||
} {
|
||||
_, err := New([]Model{model("x", p)})
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", p.Name)
|
||||
}
|
||||
if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) {
|
||||
t.Errorf("tp 4 should point at gpus_per_node: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user