Let the upgrade repair what the field's catalog already holds
tp on a llama.cpp GPU profile is read as gpus_per_node rather than refused, and a Hugging Face repository page as a vLLM source as the repository id, so models written that way are placed after the upgrade instead of dropped from the catalog. Engines adopted after an upgrade are checked for computing on their GPUs too. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
86e9a1cb83
commit
43d7c4eac1
4 files changed
+83
-25
No files matched your search
@@ -305,6 +305,16 @@ func (m Model) normalize() Model {
|
||||
if p.Nodes <= 0 {
|
||||
p.Nodes = 1
|
||||
}
|
||||
// tp on a llama.cpp GPU profile, written before gpus_per_node meant
|
||||
// anything for it, is read as the width it was meant as: N cards of
|
||||
// one node. Refusing it would have dropped a catalog's model on the
|
||||
// upgrade that made the width work. Only both given, disagreeing, is
|
||||
// refused below, as ambiguous.
|
||||
if p.Engine == "llamacpp" && p.GPUClass != "" && (p.TP > 1 || p.PP > 1) && p.Nodes == 1 {
|
||||
if n := p.TP * max(p.PP, 1); p.GPUsPerNode == 0 || p.GPUsPerNode == n {
|
||||
p.GPUsPerNode, p.TP, p.PP = n, 1, 0
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case p.CPUOnly():
|
||||
// Nothing to divide up: this profile asks for no cards at all.
|
||||
@@ -323,6 +333,13 @@ func (m Model) normalize() Model {
|
||||
p.Weight = 1
|
||||
}
|
||||
}
|
||||
// A Hugging Face repository page for a model vLLM or SGLang runs is the
|
||||
// repository it names: the engine downloads a repository id itself,
|
||||
// where the page was fetched as one file of HTML and failed at start.
|
||||
if repo, ok := repositoryPage(m.Source); ok && len(m.Profiles) > 0 &&
|
||||
!slices.ContainsFunc(m.Profiles, func(p Profile) bool { return p.Engine == "llamacpp" }) {
|
||||
m.Source = repo
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -335,9 +352,11 @@ func (m Model) validate() error {
|
||||
}
|
||||
if repo, ok := repositoryPage(m.Source); ok {
|
||||
// A page, not weights: a node fetches the HTML as one file and the
|
||||
// engine fails at start, on a path that looks right.
|
||||
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+
|
||||
"mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself",
|
||||
// engine fails at start, on a path that looks right. For vLLM it was
|
||||
// read as the repository id when the model was made; only a model
|
||||
// llama.cpp runs is left here, which needs a file.
|
||||
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not a file; "+
|
||||
"llama.cpp needs a .gguf: mirror one and use its source (repository %s)",
|
||||
m.Name, m.Source, repo)
|
||||
}
|
||||
if err := validateAdapters(m); err != nil {
|
||||
@@ -398,8 +417,9 @@ func (m Model) validate() error {
|
||||
return fmt.Errorf("model %q profile %q: llama.cpp does not span nodes; use vllm or sglang for a %d-node replica",
|
||||
m.Name, p.Name, p.Nodes)
|
||||
case p.TP > 1 || p.PP > 1:
|
||||
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, not by tp or pp: "+
|
||||
"write gpus_per_node: %d and leave tp and pp out", m.Name, p.Name, max(p.TP, 1)*max(p.PP, 1))
|
||||
return fmt.Errorf("model %q profile %q: llama.cpp splits a model across a node's GPUs by layer, and "+
|
||||
"tp %d does not agree with gpus_per_node %d: give gpus_per_node alone", m.Name, p.Name,
|
||||
max(p.TP, 1)*max(p.PP, 1), p.GPUsPerNode)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -332,37 +332,56 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
|
||||
if pl := m.PlacementProfiles(); len(pl) != 1 || pl[0].GPUsPerNode != 4 {
|
||||
t.Errorf("placement was not asked for four GPUs: %+v", pl)
|
||||
}
|
||||
// tp is a width llama.cpp has no way to use; refused, saying what to
|
||||
// write instead.
|
||||
// tp, written before gpus_per_node meant anything for llama.cpp, is read
|
||||
// as the same width rather than refused: refusing dropped the model on
|
||||
// the upgrade that made the width work.
|
||||
old := model("old", Profile{Name: "t4", Engine: "llamacpp", GPUClass: "t4", TP: 4})
|
||||
c, err = New([]Model{old})
|
||||
if err != nil {
|
||||
t.Fatalf("tp 4 on llama.cpp: %v", err)
|
||||
}
|
||||
got, _ := c.Resolve("old")
|
||||
if p := got.PlacementProfiles()[0]; p.GPUsPerNode != 4 {
|
||||
t.Errorf("tp 4 should place on four GPUs: %+v", p)
|
||||
}
|
||||
// Both given and disagreeing is ambiguous, and refused; so is more than
|
||||
// one node.
|
||||
for _, p := range []Profile{
|
||||
{Name: "tp", Engine: "llamacpp", GPUClass: "t4", TP: 4},
|
||||
{Name: "both", Engine: "llamacpp", GPUClass: "t4", TP: 4, GPUsPerNode: 2},
|
||||
{Name: "nodes", Engine: "llamacpp", GPUClass: "t4", Nodes: 2, GPUsPerNode: 1},
|
||||
} {
|
||||
_, err := New([]Model{model("x", p)})
|
||||
if err == nil {
|
||||
if _, err := New([]Model{model("x", p)}); err == nil {
|
||||
t.Errorf("%s: accepted", p.Name)
|
||||
}
|
||||
if p.TP == 4 && (err == nil || !strings.Contains(err.Error(), "gpus_per_node: 4")) {
|
||||
t.Errorf("tp 4 should point at gpus_per_node: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryPageIsNotASource(t *testing.T) {
|
||||
func TestARepositoryPageIsReadAsTheRepositoryForVLLM(t *testing.T) {
|
||||
// The field: an AWQ model's page given as a vLLM source, fetched as one
|
||||
// file of HTML. vLLM downloads a repository id itself.
|
||||
for _, src := range []string{
|
||||
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ",
|
||||
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main",
|
||||
} {
|
||||
m := model("awq")
|
||||
m.Source = src
|
||||
_, err := New([]Model{m})
|
||||
if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) {
|
||||
t.Errorf("%s: %v", src, err)
|
||||
c, err := New([]Model{m})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", src, err)
|
||||
}
|
||||
if got, _ := c.Resolve("awq"); got.Source != "Qwen/Qwen2.5-32B-Instruct-AWQ" {
|
||||
t.Errorf("%s became %q", src, got.Source)
|
||||
}
|
||||
}
|
||||
m := model("file")
|
||||
m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf"
|
||||
if _, err := New([]Model{m}); err != nil {
|
||||
// llama.cpp needs a file, which a page is not.
|
||||
g := model("gguf", Profile{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 4})
|
||||
g.Source = "https://huggingface.co/Qwen/X-GGUF"
|
||||
if _, err := New([]Model{g}); err == nil || !strings.Contains(err.Error(), "needs a .gguf") {
|
||||
t.Errorf("a page for llama.cpp: %v", err)
|
||||
}
|
||||
f := model("file")
|
||||
f.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf"
|
||||
if _, err := New([]Model{f}); err != nil {
|
||||
t.Errorf("a file under /resolve/ is a source: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user