diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f5266b..7ad710d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,17 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### llama.cpp on several GPUs of one node + +A llama.cpp profile takes several of a node's GPUs with `gpus_per_node: N`: +placement reserves N cards of its `gpu_class` on one node, the engine sees +those N (`CUDA_VISIBLE_DEVICES`) and spreads the layers over them +(`--split-mode layer`, in proportion to each card's free memory, which for +whole cards is even). `tp` or `pp` on a llama.cpp profile was a width nothing +placed — the replica stayed unplaced or ran on one card — and is now refused +when the model is added, saying to write `gpus_per_node` instead; so is +`nodes` above one. + ## v0.10.6 — 2026-10-01 Published first as release candidates v0.10.6-rc.1 to rc.8; rc.5 failed its diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 69a87d6..9ca2253 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -67,25 +67,25 @@ func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true } // Validate checks the profile against the GPUs it would run on. // -// The two refusals here are about not pretending: llama.cpp has no equivalent -// of a tensor-parallel group spanning machines, and its multi-GPU support is a -// different mechanism (--split-mode, --tensor-split) that this adapter does not -// drive. Accepting either would produce a replica that starts and then serves -// from one machine while the catalog believes it has several. +// llama.cpp spreads a model over a node's GPUs by layer (--split-mode layer), +// so gpus_per_node is how many it gets. It has no tensor parallelism and no +// group spanning machines, and accepting tp or nodes would produce a replica +// that serves from fewer cards, or one machine, while the catalog believes +// otherwise. func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { - if err := validateCommon(p, gpus); err != nil { - return err - } if p.Nodes > 1 { return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica", p.Name, p.Nodes) } - if p.TP > 1 { - return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+ - "it splits a model across local GPUs with --tensor-split, which this adapter does not drive", - p.Name) + if p.TP > 1 || p.PP > 1 { + return fmt.Errorf("profile %q: llamacpp splits a model across a node's GPUs by layer, not by tp or pp; "+ + "write gpus_per_node: %d instead", p.Name, max(p.TP, 1)*max(p.PP, 1)) } - return nil + // The common checks count a replica's GPUs as tp; for llama.cpp the + // count is gpus_per_node. + q := p + q.TP = max(p.GPUsPerNode, 1) + return validateCommon(q, gpus) } // Commands returns the processes for one shard, which is always one process. @@ -134,6 +134,13 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") { args = append(args, "--n-gpu-layers", "999") } + // Several GPUs: the layers spread across all of them, which is the only + // split llama.cpp does without a matched interconnect. In proportion to + // each card's free memory, which for the whole cards placement gives is + // an even split. + if len(s.GPUIndices) > 1 && !hasFlag(p.Args, "--split-mode", "-sm") { + args = append(args, "--split-mode", "layer") + } if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") { // What placement reserved first: it is what the node's other // replicas were placed around. diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 4f967e3..07ec11b 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -334,3 +334,30 @@ func TestDevicesAreReadFromListDevices(t *testing.T) { t.Errorf("no devices: %q", got) } } + +func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) { + four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"} + if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil { + t.Fatalf("four GPUs on a four-GPU node: %v", err) + } + if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil { + t.Error("four GPUs were accepted on a node with two") + } + + s := llamaSpec([]int{0, 1, 2, 3}) + s.Profile.GPUsPerNode = 4 + cmds, err := (&LlamaCpp{}).Commands(s) + if err != nil { + t.Fatal(err) + } + if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" { + t.Errorf("--split-mode %q over four GPUs", v) + } + if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") { + t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env) + } + one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) + if hasFlag(one[0].Args, "--split-mode") { + t.Error("a one-GPU replica was given a split mode") + } +} diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 9fa2713..99afffb 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -381,6 +381,21 @@ func (m Model) validate() error { } continue } + // llama.cpp on GPUs splits a model across one node's cards by layer, + // and how many is gpus_per_node. Not tp: llama.cpp has no tensor + // parallelism, and a tp it was given was a width nothing placed — the + // replica stayed unplaced, or ran on one card while four were meant. + if p.Engine == "llamacpp" { + switch { + case p.Nodes > 1: + return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica", + m.Name, p.Name, p.Nodes) + case p.TP > 1 || p.PP > 1: + return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+ + "write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1)) + } + continue + } // The arithmetic has to add up, or the engine is handed a shard spec it // cannot satisfy and fails minutes later during model loading. if want := p.GPUsPerNode * p.Nodes; want != p.TP*max(p.PP, 1) { diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index 619a1c6..dcc9c9c 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -321,3 +321,29 @@ func TestAProfileCannotSetWhatRunsCodeInTheEngine(t *testing.T) { t.Errorf("ordinary tuning refused: %v", err) } } + +func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) { + four := model("big", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", GPUsPerNode: 4}) + c, err := New([]Model{four}) + if err != nil { + t.Fatalf("gpus_per_node 4 on llama.cpp: %v", err) + } + m, _ := c.Resolve("big") + if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 { + t.Errorf("placement was not asked for four GPUs: %+v", pl) + } + // tp is a width llama.cpp has no way to use; refused, saying what to + // write instead. + for _, p := range []Profile{ + {Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4}, + {Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1}, + } { + _, err := New([]Model{model("x", p)}) + if err == nil { + t.Errorf("%s: accepted", p.Name) + } + if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) { + t.Errorf("tp 4 should point at gpus_per_node: %v", err) + } + } +}