diff --git a/CHANGELOG.md b/CHANGELOG.md index 43663e2..de27a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ reported ready while its GPU sat reserved and idle. Three checks now stop it: (`llama-server --list-devices`). None, on a node with GPUs, is a failed install, with what `ldd` says `libggml-cuda.so` is missing. The binary's own `ldd` could not say it: the backend is opened at run time. +- **After the upgrade**, engines the agent finds already running are held to + the same check when it adopts them: one serving a GPU placement on the CPU + is stopped, and its replacement starts through the check. The installed + CUDA build is checked again within ten minutes, and installed afresh if it + sees no GPU. - **Starting** a shard on GPUs is ready only when its process holds memory on each of them (`nvidia-smi --query-compute-apps`, its process group included). One that does not is stopped and reported failed: "placed on GPU @@ -68,8 +73,10 @@ file of HTML. it and the manifest carries it; a node checks every file of a set against it before using the set, and fetches one that differs again. - **`llmgw mirror add` takes several URLs** into one version. -- **A Hugging Face repository page is refused as a source**, saying to mirror - it or to give the repository id for vLLM to fetch itself. +- **A Hugging Face repository page given as a source** is read as the + repository id for a model vLLM or SGLang runs, which the engine downloads + itself (the node then needs to reach Hugging Face), and refused for one + llama.cpp runs, which needs a file. With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for such a model, as before. @@ -201,9 +208,10 @@ placement reserves N cards of its `gpu_class` on one node, the engine sees those N (`CUDA_VISIBLE_DEVICES`) and spreads the layers over them (`--split-mode layer`, in proportion to each card's free memory, which for whole cards is even). `tp` or `pp` on a llama.cpp profile was a width nothing -placed — the replica stayed unplaced or ran on one card — and is now refused -when the model is added, saying to write `gpus_per_node` instead; so is -`nodes` above one. +placed — the replica stayed unplaced or ran on one card. It is now read as +`gpus_per_node`, so a profile written that way is placed on that many cards +after the upgrade, rather than dropped; only both given and disagreeing, or +`nodes` above one, is refused. ## v0.10.6 — 2026-10-01 diff --git a/internal/agent/agent.go b/internal/agent/agent.go index a522901..3cb8ea2 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -314,6 +314,17 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { "shard", o.ShardID, "replica", o.ReplicaID, "err", err) return } + // An engine started by an agent older than the GPU check may be one + // serving a GPU placement on the CPU. Stopped, so its replacement starts + // through the check and fails there in words, if the build still sees + // no GPU — rather than going on unnoticed after the upgrade that could + // have caught it. + if err := a.onItsGPUs(ctx, eng, o.Spec, sh.Status().PID); err != nil { + a.log.Error("stopping an adopted engine that computes on the CPU though placed on GPUs", + "shard", o.ShardID, "replica", o.ReplicaID, "err", err) + _ = a.sup.Stop(ctx, o.ShardID) + return + } a.log.Info("adopted engine is serving", "shard", o.ShardID, "replica", o.ReplicaID) } diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 66bea23..70c599e 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -305,6 +305,16 @@ func (m Model) normalize() Model { if p.Nodes <= 0 { p.Nodes = 1 } + // tp on a llama.cpp GPU profile, written before gpus_per_node meant + // anything for it, is read as the width it was meant as: N cards of + // one node. Refusing it would have dropped a catalog's model on the + // upgrade that made the width work. Only both given, disagreeing, is + // refused below, as ambiguous. + if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 { + if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n { + p.GPUsPerNode, p.TP, p.PP = n, 1, 0 + } + } switch { case p.CPUOnly(): // Nothing to divide up: this profile asks for no cards at all. @@ -323,6 +333,13 @@ func (m Model) normalize() Model { p.Weight = 1 } } + // A Hugging Face repository page for a model vLLM or SGLang runs is the + // repository it names: the engine downloads a repository id itself, + // where the page was fetched as one file of HTML and failed at start. + if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 && + !slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) { + m.Source = repo + } return m } @@ -335,9 +352,11 @@ func (m Model) validate() error { } if repo, ok := repositoryPage(m.Source); ok { // A page, not weights: a node fetches the HTML as one file and the - // engine fails at start, on a path that looks right. - return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+ - "mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself", + // engine fails at start, on a path that looks right. For vLLM it was + // read as the repository id when the model was made; only a model + // llama.cpp runs is left here, which needs a file. + return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+ + "llama.cpp needs a .gguf: mirror one and use its source (repository %s)", m.Name, m.Source, repo) } if err := validateAdapters(m); err != nil { @@ -398,8 +417,9 @@ func (m Model) validate() error { return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica", m.Name, p.Name, p.Nodes) case p.TP > 1 || p.PP > 1: - return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+ - "write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1)) + return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+ + "tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name, + max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode) } continue } diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index 6829d78..255e117 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -332,37 +332,56 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) { if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 { t.Errorf("placement was not asked for four GPUs: %+v", pl) } - // tp is a width llama.cpp has no way to use; refused, saying what to - // write instead. + // tp, written before gpus_per_node meant anything for llama.cpp, is read + // as the same width rather than refused: refusing dropped the model on + // the upgrade that made the width work. + old := model("old", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", TP: 4}) + c, err = New([]Model{old}) + if err != nil { + t.Fatalf("tp 4 on llama.cpp: %v", err) + } + got, _ := c.Resolve("old") + if p := got.PlacementProfiles()[0]; p.GPUsPerNode != 4 { + t.Errorf("tp 4 should place on four GPUs: %+v", p) + } + // Both given and disagreeing is ambiguous, and refused; so is more than + // one node. for _, p := range []Profile{ - {Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4}, + {Name: "both", Engine: "llamacpp", GPUClass: "t4", TP: 4, GPUsPerNode: 2}, {Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1}, } { - _, err := New([]Model{model("x", p)}) - if err == nil { + if _, err := New([]Model{model("x", p)}); err == nil { t.Errorf("%s: accepted", p.Name) } - if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) { - t.Errorf("tp 4 should point at gpus_per_node: %v", err) - } } } -func TestARepositoryPageIsNotASource(t *testing.T) { +func TestARepositoryPageIsReadAsTheRepositoryForVLLM(t *testing.T) { + // The field: an AWQ model's page given as a vLLM source, fetched as one + // file of HTML. vLLM downloads a repository id itself. for _, src := range []string{ "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ", "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main", } { m := model("awq") m.Source = src - _, err := New([]Model{m}) - if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) { - t.Errorf("%s: %v", src, err) + c, err := New([]Model{m}) + if err != nil { + t.Fatalf("%s: %v", src, err) + } + if got, _ := c.Resolve("awq"); got.Source != "Qwen/Qwen2.5-32B-Instruct-AWQ" { + t.Errorf("%s became %q", src, got.Source) } } - m := model("file") - m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf" - if _, err := New([]Model{m}); err != nil { + // llama.cpp needs a file, which a page is not. + g := model("gguf", Profile{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4}) + g.Source = "https://huggingface.co/Qwen/X-GGUF" + if _, err := New([]Model{g}); err == nil || !strings.Contains(err.Error(), "needs a .gguf") { + t.Errorf("a page for llama.cpp: %v", err) + } + f := model("file") + f.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf" + if _, err := New([]Model{f}); err != nil { t.Errorf("a file under /resolve/ is a source: %v", err) } }