From 43d7c4eac1324e2872f0b79aea519e708b23229a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 14:20:11 +0200 Subject: [PATCH] Let the upgrade repair what the field's catalog already holds tp on a llama.cpp GPU profile is read as gpus_per_node rather than refused, and a Hugging Face repository page as a vLLM source as the repository id, so models written that way are placed after the upgrade instead of dropped from the catalog. Engines adopted after an upgrade are checked for computing on their GPUs too. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 ++++++--- internal/agent/agent.go | 11 ++++++ internal/control/catalog/catalog.go | 30 ++++++++++++--- internal/control/catalog/catalog_test.go | 49 ++++++++++++++++-------- 4 files changed, 83 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43663e2..de27a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ reported ready while its GPU sat reserved and idle. Three checks now stop it: (`llama-server --list-devices`). None, on a node with GPUs, is a failed install, with what `ldd` says `libggml-cuda.so` is missing. The binary's own `ldd` could not say it: the backend is opened at run time. +- **After the upgrade**, engines the agent finds already running are held to + the same check when it adopts them: one serving a GPU placement on the CPU + is stopped, and its replacement starts through the check. The installed + CUDA build is checked again within ten minutes, and installed afresh if it + sees no GPU. - **Starting** a shard on GPUs is ready only when its process holds memory on each of them (`nvidia-smi --query-compute-apps`, its process group included). One that does not is stopped and reported failed: "placed on GPU @@ -68,8 +73,10 @@ file of HTML. it and the manifest carries it; a node checks every file of a set against it before using the set, and fetches one that differs again. - **`llmgw mirror add` takes several URLs** into one version. -- **A Hugging Face repository page is refused as a source**, saying to mirror - it or to give the repository id for vLLM to fetch itself. +- **A Hugging Face repository page given as a source** is read as the + repository id for a model vLLM or SGLang runs, which the engine downloads + itself (the node then needs to reach Hugging Face), and refused for one + llama.cpp runs, which needs a file. With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for such a model, as before. @@ -201,9 +208,10 @@ placement reserves N cards of its `gpu_class` on one node, the engine sees those N (`CUDA_VISIBLE_DEVICES`) and spreads the layers over them (`--split-mode layer`, in proportion to each card's free memory, which for whole cards is even). `tp` or `pp` on a llama.cpp profile was a width nothing -placed — the replica stayed unplaced or ran on one card — and is now refused -when the model is added, saying to write `gpus_per_node` instead; so is -`nodes` above one. +placed — the replica stayed unplaced or ran on one card. It is now read as +`gpus_per_node`, so a profile written that way is placed on that many cards +after the upgrade, rather than dropped; only both given and disagreeing, or +`nodes` above one, is refused. ## v0.10.6 — 2026-10-01 diff --git a/internal/agent/agent.go b/internal/agent/agent.go index a522901..3cb8ea2 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -314,6 +314,17 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { "shard", o.ShardID, "replica", o.ReplicaID, "err", err) return } + // An engine started by an agent older than the GPU check may be one + // serving a GPU placement on the CPU. Stopped, so its replacement starts + // through the check and fails there in words, if the build still sees + // no GPU — rather than going on unnoticed after the upgrade that could + // have caught it. + if err := a.onItsGPUs(ctx, eng, o.Spec, sh.Status().PID); err != nil { + a.log.Error("stopping an adopted engine that computes on the CPU though placed on GPUs", + "shard", o.ShardID, "replica", o.ReplicaID, "err", err) + _ = a.sup.Stop(ctx, o.ShardID) + return + } a.log.Info("adopted engine is serving", "shard", o.ShardID, "replica", o.ReplicaID) } diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 66bea23..70c599e 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -305,6 +305,16 @@ func (m Model) normalize() Model { if p.Nodes <= 0 { p.Nodes = 1 } + // tp on a llama.cpp GPU profile, written before gpus_per_node meant + // anything for it, is read as the width it was meant as: N cards of + // one node. Refusing it would have dropped a catalog's model on the + // upgrade that made the width work. Only both given, disagreeing, is + // refused below, as ambiguous. + if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 { + if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n { + p.GPUsPerNode, p.TP, p.PP = n, 1, 0 + } + } switch { case p.CPUOnly(): // Nothing to divide up: this profile asks for no cards at all. @@ -323,6 +333,13 @@ func (m Model) normalize() Model { p.Weight = 1 } } + // A Hugging Face repository page for a model vLLM or SGLang runs is the + // repository it names: the engine downloads a repository id itself, + // where the page was fetched as one file of HTML and failed at start. + if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 && + !slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) { + m.Source = repo + } return m } @@ -335,9 +352,11 @@ func (m Model) validate() error { } if repo, ok := repositoryPage(m.Source); ok { // A page, not weights: a node fetches the HTML as one file and the - // engine fails at start, on a path that looks right. - return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+ - "mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself", + // engine fails at start, on a path that looks right. For vLLM it was + // read as the repository id when the model was made; only a model + // llama.cpp runs is left here, which needs a file. + return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+ + "llama.cpp needs a .gguf: mirror one and use its source (repository %s)", m.Name, m.Source, repo) } if err := validateAdapters(m); err != nil { @@ -398,8 +417,9 @@ func (m Model) validate() error { return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes) case p.TP > 1 || p.PP > 1: - return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+ - "write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1)) + return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+ + "tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name, + max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode) } continue } diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index 6829d78..255e117 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -332,37 +332,56 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) { if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 { t.Errorf("placement was not asked for four GPUs: %+v", pl) } - // tp is a width llama.cpp has no way to use; refused, saying what to - // write instead. + // tp, written before gpus_per_node meant anything for llama.cpp, is read + // as the same width rather than refused: refusing dropped the model on + // the upgrade that made the width work. + old := model("old", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", TP: 4}) + c, err = New([]Model{old}) + if err != nil { + t.Fatalf("tp 4 on llama.cpp: %v", err) + } + got, _ := c.Resolve("old") + if p := got.PlacementProfiles()[0]; p.GPUsPerNode != 4 { + t.Errorf("tp 4 should place on four GPUs: %+v", p) + } + // Both given and disagreeing is ambiguous, and refused; so is more than + // one node. for _, p := range []Profile{ - {Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4}, + {Name: "both", Engine: "llamacpp", GPUClass: "t4", TP: 4, GPUsPerNode: 2}, {Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1}, } { - _, err := New([]Model{model("x", p)}) - if err == nil { + if _, err := New([]Model{model("x", p)}); err == nil { t.Errorf("%s: accepted", p.Name) } - if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) { - t.Errorf("tp 4 should point at gpus_per_node: %v", err) - } } } -func TestARepositoryPageIsNotASource(t *testing.T) { +func TestARepositoryPageIsReadAsTheRepositoryForVLLM(t *testing.T) { + // The field: an AWQ model's page given as a vLLM source, fetched as one + // file of HTML. vLLM downloads a repository id itself. for _, src := range []string{ "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ", "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main", } { m := model("awq") m.Source = src - _, err := New([]Model{m}) - if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) { - t.Errorf("%s: %v", src, err) + c, err := New([]Model{m}) + if err != nil { + t.Fatalf("%s: %v", src, err) + } + if got, _ := c.Resolve("awq"); got.Source != "Qwen/Qwen2.5-32B-Instruct-AWQ" { + t.Errorf("%s became %q", src, got.Source) } } - m := model("file") - m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf" - if _, err := New([]Model{m}); err != nil { + // llama.cpp needs a file, which a page is not. + g := model("gguf", Profile{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}) + g.Source = "https://huggingface.co/Qwen/X-GGUF" + if _, err := New([]Model{g}); err == nil || !strings.Contains(err.Error(), "needs a .gguf") { + t.Errorf("a page for llama.cpp: %v", err) + } + f := model("file") + f.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf" + if _, err := New([]Model{f}); err != nil { t.Errorf("a file under /resolve/ is a source: %v", err) } }