Refuse weights the nodes' llama.cpp cannot load, before fetching them
kev-0.5b declared architecture "ggmlc", which llama.cpp does not know. It was fetched, catalogued and placed, and failed on every node — while the answer was in the first kilobytes of the file. internal/gguf reads a GGUF header (v1-v3) up to general.architecture, over HTTP with a Range request, and judges it against the architectures llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at that tag); a vision projector is not a model on its own. An unknown build is not refused, only not vouched for, and a header that cannot be fetched does not refuse anything. Checked in three places: the registry listing (from the architecture Hugging Face reports with expand[]=gguf, so one request per repository) shows it and withholds the fetch button; a fetch reads the header and refuses unless "fetch anyway"; and the model service refuses to add such a model, which covers the serve form and ctl model add. Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3 (bert), and Qwen2.5-0.5B on Hugging Face through its redirect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c609805e4b
commit
f13b18149a
12 files changed
+782
-3
No files matched your search
@@ -0,0 +1,52 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
func ggufAt(t *testing.T, arch string) string {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(1))
|
||||
str("general.architecture")
|
||||
w(uint32(8))
|
||||
str(arch)
|
||||
body := b.Bytes()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL + "/m.gguf"
|
||||
}
|
||||
|
||||
func TestAModelTheNodesCannotLoadIsNotCatalogued(t *testing.T) {
|
||||
check := servableCheck("b10991")
|
||||
llama := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
|
||||
|
||||
err := check(context.Background(), catalog.Model{Name: "kev", Source: ggufAt(t, "ggmlc"), Profiles: llama})
|
||||
if err == nil || !strings.Contains(err.Error(), "ggmlc") {
|
||||
t.Errorf("a ggmlc model was accepted: %v", err)
|
||||
}
|
||||
if err := check(context.Background(), catalog.Model{Name: "q", Source: ggufAt(t, "qwen2"), Profiles: llama}); err != nil {
|
||||
t.Errorf("a qwen2 model was refused: %v", err)
|
||||
}
|
||||
// Not llama.cpp's to judge: vLLM reads a repository, not a GGUF header.
|
||||
vllm := []catalog.Profile{{Name: "gpu", Engine: "vllm"}}
|
||||
if err := check(context.Background(), catalog.Model{Name: "v", Source: ggufAt(t, "ggmlc"), Profiles: vllm}); err != nil {
|
||||
t.Errorf("a vLLM profile was judged by llama.cpp's list: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user