Refuse weights the nodes' llama.cpp cannot load, before fetching them
kev-0.5b declared architecture "ggmlc", which llama.cpp does not know. It was fetched, catalogued and placed, and failed on every node — while the answer was in the first kilobytes of the file. internal/gguf reads a GGUF header (v1-v3) up to general.architecture, over HTTP with a Range request, and judges it against the architectures llama.cpp b10991 lists in LLM_ARCH_NAMES (152, copied from the source at that tag); a vision projector is not a model on its own. An unknown build is not refused, only not vouched for, and a header that cannot be fetched does not refuse anything. Checked in three places: the registry listing (from the architecture Hugging Face reports with expand[]=gguf, so one request per repository) shows it and withholds the fetch button; a fetch reads the header and refuses unless "fetch anyway"; and the model service refuses to add such a model, which covers the serve form and ctl model add. Tried on real files: the mirror's qwen2.5-1.5b (qwen2) and bge-m3 (bert), and Qwen2.5-0.5B on Hugging Face through its redirect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c609805e4b
commit
f13b18149a
12 files changed
+782
-3
No files matched your search
@@ -67,6 +67,9 @@ type AccountPages struct {
|
||||
personalDefaults *store.Limits
|
||||
// orgs is set when /org is mounted, so the menu can offer it to managers.
|
||||
orgs OrgViewer
|
||||
// llamaVersion is the llama.cpp build nodes install, for judging whether a
|
||||
// GGUF can be served before it is fetched or catalogued.
|
||||
llamaVersion string
|
||||
// priced says whether a budget can ever be spent against.
|
||||
priced Priced
|
||||
// registry browses a model registry, or is nil where there is no way out.
|
||||
@@ -602,3 +605,6 @@ func (p *AccountPages) renderWithCSRF(w http.ResponseWriter, r *http.Request,
|
||||
p.fillPlaces(r.Context(), user, &d)
|
||||
render(w, "profile", d)
|
||||
}
|
||||
|
||||
// SetLlamaCppVersion tells the pages which llama.cpp build the nodes run.
|
||||
func (p *AccountPages) SetLlamaCppVersion(v string) { p.llamaVersion = v }
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -41,6 +42,10 @@ type Registry interface {
|
||||
|
||||
// repoSet is one thing worth fetching: a file, or every shard of one.
|
||||
type repoSet struct {
|
||||
// Verdict is whether the nodes' llama.cpp can load it, from the
|
||||
// architecture the registry reports; a set that cannot be served is
|
||||
// shown with the reason and no fetch button.
|
||||
Verdict gguf.Verdict
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
@@ -179,6 +184,15 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
model := strings.TrimSpace(r.FormValue("model"))
|
||||
version := strings.TrimSpace(r.FormValue("version"))
|
||||
|
||||
// Read before anything is downloaded: the header says whether the
|
||||
// nodes can load it, and a file that cannot be served is an hour
|
||||
// of download and a replica failing on every node.
|
||||
if r.FormValue("force") == "" {
|
||||
if v := p.checkSource(ctx, strings.TrimSpace(sources[0])); !v.Servable {
|
||||
return "", fmt.Errorf("not fetched: %s. Tick \"fetch anyway\" to take it regardless", v.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
var started int
|
||||
for i, src := range sources {
|
||||
src = strings.TrimSpace(src)
|
||||
@@ -379,7 +393,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
continue
|
||||
}
|
||||
form := planServe(d.Rows[i].Status,
|
||||
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog)
|
||||
byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog, func(src string) gguf.Verdict { return p.checkSource(r.Context(), src) })
|
||||
d.Rows[i].Serve = &form
|
||||
break
|
||||
}
|
||||
@@ -467,6 +481,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
// the same branch lands somewhere else.
|
||||
Version: got.ShortRevision(),
|
||||
}
|
||||
rs.Verdict = p.verdictFor(got.Architecture, set.Name)
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
@@ -498,3 +513,25 @@ func defaultModelName(repoID string) string {
|
||||
name = strings.TrimSuffix(name, "-gguf")
|
||||
return strings.ToLower(name)
|
||||
}
|
||||
|
||||
// verdictFor judges a registry file set by the repository's architecture. A
|
||||
// vision projector is its own architecture whatever the repository says.
|
||||
func (p *AccountPages) verdictFor(arch, name string) gguf.Verdict {
|
||||
if strings.Contains(strings.ToLower(name), "mmproj") {
|
||||
return gguf.LlamaCpp(p.llamaVersion, "clip")
|
||||
}
|
||||
if arch == "" || !gguf.IsGGUF(name) {
|
||||
return gguf.Verdict{Servable: true}
|
||||
}
|
||||
return gguf.LlamaCpp(p.llamaVersion, arch)
|
||||
}
|
||||
|
||||
// checkSource reads a GGUF source's header and judges it; anything else, or a
|
||||
// header that cannot be read, passes.
|
||||
func (p *AccountPages) checkSource(ctx context.Context, source string) gguf.Verdict {
|
||||
url, ok := sourcecheck.HTTPSource(source)
|
||||
if !ok || !gguf.IsGGUF(url) || p.llamaVersion == "" {
|
||||
return gguf.Verdict{Servable: true}
|
||||
}
|
||||
return gguf.Check(ctx, url, p.llamaVersion)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -60,7 +61,7 @@ type serveForm struct {
|
||||
|
||||
// planServe works out what serving this version would mean.
|
||||
func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
fleet Source, cat CatalogAdmin,
|
||||
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
|
||||
) serveForm {
|
||||
f := serveForm{
|
||||
Model: st.Model,
|
||||
@@ -99,6 +100,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
return f
|
||||
case "llamacpp":
|
||||
f.Profile = "cpu"
|
||||
if check != nil {
|
||||
if v := check(f.Source); !v.Servable {
|
||||
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
|
||||
return f
|
||||
}
|
||||
}
|
||||
default:
|
||||
f.Profile = "gpu"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ggufServer serves a GGUF whose header names an architecture.
|
||||
func ggufServer(t *testing.T, arch string) string {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(2))
|
||||
str("general.architecture")
|
||||
w(uint32(8))
|
||||
str(arch)
|
||||
str("general.name")
|
||||
w(uint32(8))
|
||||
str("m")
|
||||
body := b.Bytes()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||
http.ServeContent(rw, r, "m.gguf", time.Time{}, bytes.NewReader(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL + "/m.gguf"
|
||||
}
|
||||
|
||||
func mirrorPageLlama(t *testing.T, d MirrorDeps) *http.ServeMux {
|
||||
t.Helper()
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
p.SetLlamaCppVersion("b10991")
|
||||
mux := http.NewServeMux()
|
||||
p.RegisterAdminMirror(mux, d)
|
||||
return mux
|
||||
}
|
||||
|
||||
// The file that started this: kev-0.5b, architecture "ggmlc", downloaded,
|
||||
// catalogued, and refused by llama.cpp on every node it was placed on.
|
||||
func TestAFileTheNodesCannotLoadIsNotFetched(t *testing.T) {
|
||||
m := &fakeMirror{}
|
||||
mux := mirrorPageLlama(t, MirrorDeps{Mirror: m})
|
||||
src := ggufServer(t, "ggmlc")
|
||||
|
||||
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {src}, "model": {"kev"}, "version": {"v1"}}).Body.String()
|
||||
if len(m.all) != 0 {
|
||||
t.Fatal("a file llama.cpp cannot load was sent to the mirror")
|
||||
}
|
||||
if !strings.Contains(body, "ggmlc") || !strings.Contains(body, "cannot load") {
|
||||
t.Errorf("the refusal should say why; got %s", body)
|
||||
}
|
||||
|
||||
// Asked for regardless, it is fetched.
|
||||
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {src}, "model": {"kev"}, "version": {"v1"}, "force": {"1"}})
|
||||
if len(m.all) != 1 {
|
||||
t.Error("fetch anyway did not fetch")
|
||||
}
|
||||
|
||||
// And one it can load goes straight through.
|
||||
m.all = nil
|
||||
mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"},
|
||||
"source": {ggufServer(t, "qwen2")}, "model": {"q"}, "version": {"v1"}})
|
||||
if len(m.all) != 1 {
|
||||
t.Error("a qwen2 GGUF was refused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRegistryListingSaysWhatCannotBeServed(t *testing.T) {
|
||||
reg := &fakeRegistry{repo: hf.Repo{ID: "someone/kev-gguf", Revision: "abc12345", Architecture: "ggmlc",
|
||||
Files: []hf.File{{Path: "kev-q4_k_m.gguf", Bytes: 500 << 20}}}}
|
||||
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
|
||||
"/admin/mirror?q=kev&repo=someone/kev-gguf").Body.String()
|
||||
if !strings.Contains(body, "cannot be served") {
|
||||
t.Error("the listing does not say the file cannot be served")
|
||||
}
|
||||
if strings.Contains(body, `value="`+"https://huggingface.co") && strings.Contains(body, ">Fetch</button>") {
|
||||
t.Error("the listing still offers to fetch it")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user