From e84631ce3a8d6268cdeb347eb5be2bcc85d0402a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 26 Oct 2026 13:00:28 +0100 Subject: [PATCH] The explorer lists only what a node loads A repository with neither GGUF nor safetensors (by the registry's summary or its tags) is hidden under "runs here", which is ticked by default, and counted with the architectures llama.cpp cannot load; unticked it is listed. An open repository's pickle files are not offered, only counted, and a repository of pickles only says to look for another version. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 5 ++- CHANGELOG.md | 5 ++- internal/web/admin_mirror.go | 24 +++++++----- internal/web/admin_mirror_test.go | 2 +- internal/web/i18n_mirror_test.go | 2 +- internal/web/locales/en/mirror.json | 17 ++++---- internal/web/locales/tr/mirror.json | 9 +++-- internal/web/mirror_explore.go | 16 ++++++++ internal/web/mirror_explore_test.go | 49 ++++++++++++++++++++++++ internal/web/templates/admin_mirror.html | 3 +- 10 files changed, 105 insertions(+), 27 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index e7d671d9..c3134dc5 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -1800,8 +1800,9 @@ and never overwritten. What did not: everything below. how the malicious models found on Hugging Face work. The node refuses such a source, or a mirror set holding one, before fetching anything; a model being added or given such a source is refused; vLLM and SGLang downloading a model -by its id are given `--load-format safetensors`; the mirror page says so on -the file. `allow_pickle: true` on a model lets it through, on purpose. On the +by its id are given `--load-format safetensors`. The explorer hides a repository with neither GGUF nor safetensors +(under "runs here", ticked by default) and does not offer a repository's +pickle files. `allow_pickle: true` on a model lets it through, on purpose. On the PoC, `triposr` (`hf:stabilityai/TripoSR/model.ckpt`) is the one model that needs it. diff --git a/CHANGELOG.md b/CHANGELOG.md index a5a87d3b..5bf6cb70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,10 @@ APIs change between minors without a deprecation period. or a mirror version with one in it, before fetching it; adding such a model is refused; vLLM and SGLang downloading a model by its id read its safetensors only; the mirror page marks the file. `allow_pickle: true` - on a model lets it through on purpose. BACKLOG M lists what is next: + on a model lets it through on purpose. The mirror's explorer no longer + lists a repository with nothing a node loads (no GGUF, no safetensors) + while "runs here" is ticked, and a repository's pickle files are not + offered, only counted. BACKLOG M lists what is next: checksums against Hugging Face everywhere, its own scan, trusted publishers, engines without the internet. diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index ae7175ec..88c19dbe 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" + weightspkg "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" @@ -175,10 +175,13 @@ type adminMirrorData struct { // Card is the open repository's model card, rendered. Card template.HTML // RepoURL is the open repository on the registry's own site. - RepoURL string - Repo *hf.Repo - RepoSets []repoSet - RegistryErr string + RepoURL string + Repo *hf.Repo + RepoSets []repoSet + // PickleHidden is how many of the open repository's weights files were + // left out for being in a pickle format, which nodes refuse. + PickleHidden int + RegistryErr string Message string Error string @@ -789,6 +792,12 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string d.Card = renderCard(got.Card, d.RepoURL+"/blob/"+revOrMain(got.Revision)+"/") var weights []hf.File for _, f := range got.Files { + // Pickles run code when loaded and a node refuses them unless a + // model says allow_pickle: not offered, only counted. + if weightspkg.Pickled(f.Path) { + d.PickleHidden++ + continue + } if !f.IsWeights() { // Thirty files of which four are worth gigabytes. The rest are // a README and a tokenizer, and listing them buries the ones @@ -877,11 +886,6 @@ func defaultModelName(repoID string) string { return hf.ModelName(repoID) } // verdictFor judges a registry file set by the repository's architecture. A // vision projector is its own architecture whatever the repository says. func (p *AccountPages) verdictFor(l *Lang, arch, name string) gguf.Verdict { - // Not refused here — the mirror keeps what it is given — but a node - // refuses to fetch it for a model without allow_pickle. - if weights.Pickled(name) { - return gguf.Verdict{Checked: true, Reason: l.T("card.pickle")} - } if strings.Contains(strings.ToLower(name), "mmproj") { return gguf.LlamaCpp(p.llamaVersion, "clip") } diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index 9f1cf638..ff8aa567 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -364,7 +364,7 @@ func (f *fakeRegistry) Repo(_ context.Context, id string) (hf.Repo, error) { func TestSearchingTheRegistryListsRepositories(t *testing.T) { reg := &fakeRegistry{results: []hf.Model{ - {ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation"}, + {ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation", Tags: []string{"gguf"}}, }} body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), "/admin/mirror?q=qwen+coder").Body.String() diff --git a/internal/web/i18n_mirror_test.go b/internal/web/i18n_mirror_test.go index 12072b82..87b9b04f 100644 --- a/internal/web/i18n_mirror_test.go +++ b/internal/web/i18n_mirror_test.go @@ -49,7 +49,7 @@ func TestTheMirrorPageSpeaksTurkishWhenAsked(t *testing.T) { {Model: "llama", Version: "v1", File: "l.gguf", State: mirror.Fetching, BytesDone: 1 << 20, BytesTotal: 4 << 20}, }} - reg := &fakeRegistry{results: []hf.Model{{ID: "Qwen/Qwen3-8B-GGUF", Downloads: 1247, + reg := &fakeRegistry{results: []hf.Model{{ID: "Qwen/Qwen3-8B-GGUF", Downloads: 1247, Tags: []string{"gguf"}, CreatedAt: time.Now().Add(-3 * time.Hour)}}} mux := mirrorPageWith(t, m, reg, true) diff --git a/internal/web/locales/en/mirror.json b/internal/web/locales/en/mirror.json index dec737bf..9ecee295 100644 --- a/internal/web/locales/en/mirror.json +++ b/internal/web/locales/en/mirror.json @@ -95,15 +95,15 @@ "explore.js.follow": "follow it under Held", "explore.js.started": "Started", "explore.js.unreachable": "Could not reach the gateway:", - "explore.hidden.one": "%d more is hidden: their architecture is one the nodes' llama.cpp cannot load.", - "explore.hidden.other": "%d more are hidden: their architecture is one the nodes' llama.cpp cannot load.", + "explore.hidden.one": "%d more is hidden: nothing in it these nodes can load — an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", + "explore.hidden.other": "%d more are hidden: nothing in them these nodes can load — an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", "explore.untick": "Untick “only what these nodes can run” to see them.", "explore.gated_note": "A gated repository needs its terms accepted with a Hugging Face account, and this browses anonymously — the files will list, and fetching one will fail at the mirror. Download it by hand and put it in the mirror directory, or give the mirror a token.", - "explore.empty.query_hidden.one": "Nothing matched “%[2]s” that these nodes can run — %[1]d result was hidden for an architecture llama.cpp cannot load.", - "explore.empty.query_hidden.other": "Nothing matched “%[2]s” that these nodes can run — %[1]d results were hidden for an architecture llama.cpp cannot load.", + "explore.empty.query_hidden.one": "Nothing matched “%[2]s” that these nodes can run — %[1]d result was hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", + "explore.empty.query_hidden.other": "Nothing matched “%[2]s” that these nodes can run — %[1]d results were hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", "explore.empty.query": "Nothing matched “%s”.", - "explore.empty.hidden.one": "Nothing to list that these nodes can run — %d result was hidden for an architecture llama.cpp cannot load.", - "explore.empty.hidden.other": "Nothing to list that these nodes can run — %d results were hidden for an architecture llama.cpp cannot load.", + "explore.empty.hidden.one": "Nothing to list that these nodes can run — %d result was hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", + "explore.empty.hidden.other": "Nothing to list that these nodes can run — %d results were hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.", "explore.empty.nothing": "Nothing to list.", "mirror.fetch.intro": "By hand, for anything the search does not reach.", @@ -137,7 +137,10 @@ "card.model": "model", "card.cannot_serve": "cannot be served", "card.imatrix": "it is an importance matrix for quantising, not a model", - "card.pickle": "it is in a pickle format, which runs code when it is loaded; nodes refuse it unless the model says allow_pickle — take the safetensors or GGUF", + "card.pickle_hidden.one": "%d file is not shown: it is in a pickle format, which runs code when it is loaded, and nodes refuse it.", + "card.pickle_hidden.other": "%d files are not shown: they are in a pickle format, which runs code when it is loaded, and nodes refuse them.", + "card.only_pickle.one": "%[2]s has weights only in a pickle format (%[1]d file), which runs code when it is loaded; nodes refuse it. Look for a safetensors or GGUF version of the model.", + "card.only_pickle.other": "%[2]s has weights only in a pickle format (%[1]d files), which runs code when it is loaded; nodes refuse them. Look for a safetensors or GGUF version of the model.", "card.fetch": "Fetch", "card.weights_note": "Only the weights are listed; a repository is mostly a README and a tokenizer. A safetensors set is fetched with the files vLLM reads beside it — config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard model is one decision, not fifteen. Two separate files are two rows because they are usually alternatives: one quantisation or the other, not both.", "card.pinned_note": "The source URLs are pinned to %s rather than to a branch, so what arrives is what is listed here even if the branch moves in between — and a single file's checksum above is what the registry publishes, to compare with what the mirror computes once it has it.", diff --git a/internal/web/locales/tr/mirror.json b/internal/web/locales/tr/mirror.json index a065c07a..ce6dae3d 100644 --- a/internal/web/locales/tr/mirror.json +++ b/internal/web/locales/tr/mirror.json @@ -93,12 +93,12 @@ "explore.js.follow": "Aynadakiler sekmesinden izleyin", "explore.js.started": "Başladı", "explore.js.unreachable": "Ağ geçidine ulaşılamadı:", - "explore.hidden.other": "%d sonuç daha gizlendi: mimarileri düğümlerdeki llama.cpp'nin yükleyemediği bir mimari.", + "explore.hidden.other": "%d sonuç daha gizlendi: düğümlerin yükleyebileceği bir şey yok — llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.", "explore.untick": "Onları görmek için “yalnızca bu düğümlerin çalıştırabildikleri” kutusunun işaretini kaldırın.", "explore.gated_note": "Erişimi kısıtlı bir depo, koşullarının bir Hugging Face hesabıyla kabul edilmesini ister; buradaki tarama ise anonimdir — dosyalar listelenir, ancak birini indirmek aynada başarısız olur. Dosyayı elle indirip ayna dizinine koyun ya da aynaya bir token verin.", - "explore.empty.query_hidden.other": "“%[2]s” ile eşleşen ve bu düğümlerin çalıştırabileceği bir şey yok — llama.cpp'nin yükleyemediği bir mimari nedeniyle %[1]d sonuç gizlendi.", + "explore.empty.query_hidden.other": "“%[2]s” ile eşleşen ve bu düğümlerin çalıştırabileceği bir şey yok — %[1]d sonuç gizlendi: llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.", "explore.empty.query": "“%s” ile eşleşen bir şey yok.", - "explore.empty.hidden.other": "Bu düğümlerin çalıştırabileceği listelenecek bir şey yok — llama.cpp'nin yükleyemediği bir mimari nedeniyle %d sonuç gizlendi.", + "explore.empty.hidden.other": "Bu düğümlerin çalıştırabileceği listelenecek bir şey yok — %d sonuç gizlendi: llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.", "explore.empty.nothing": "Listelenecek bir şey yok.", "mirror.fetch.intro": "Aramanın ulaşamadığı her şey için, elle.", @@ -132,7 +132,8 @@ "card.model": "model", "card.cannot_serve": "sunulamaz", "card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi", - "card.pickle": "pickle biçiminde; yüklenirken kod çalıştırır. Model allow_pickle demedikçe düğümler onu almaz — safetensors ya da GGUF olanı seçin", + "card.pickle_hidden.other": "%d dosya gösterilmiyor: pickle biçiminde, yüklenirken kod çalıştırır ve düğümler onu almaz.", + "card.only_pickle.other": "%[2]s deposunun ağırlıkları yalnızca pickle biçiminde (%[1]d dosya); yüklenirken kod çalıştırır, düğümler almaz. Modelin safetensors ya da GGUF sürümünü arayın.", "card.fetch": "İndir", "card.weights_note": "Yalnızca ağırlıklar listelenir; bir deponun çoğu bir README ve bir tokenizer'dır. Bir safetensors kümesi, vLLM'in yanında okuduğu dosyalarla — config.json, tokenizer, parça dizini — birlikte indirilir, asla kodla birlikte değil. Bir dosyanın parçaları tek satır ve tek düğmedir — on beş parçalı bir model on beş değil, tek bir karardır. Ayrı iki dosya iki satırdır, çünkü genellikle birbirinin seçeneğidir: ya bir nicemleme ya öteki, ikisi birden değil.", "card.pinned_note": "Kaynak URL'ler bir dala değil %s sürümüne sabitlenmiştir; böylece dal arada değişse bile gelen, burada listelenendir — yukarıdaki tek dosya sağlama toplamı da model deposunun yayımladığıdır ve ayna dosyayı aldığında hesapladığıyla karşılaştırılabilir.", diff --git a/internal/web/mirror_explore.go b/internal/web/mirror_explore.go index 607ff2e1..496434b5 100644 --- a/internal/web/mirror_explore.go +++ b/internal/web/mirror_explore.go @@ -5,6 +5,7 @@ import ( "html/template" "net/http" "net/url" + "slices" "strings" "time" @@ -286,6 +287,12 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) { if e.Here && m.Params() > 0 && m.Params() < minParams { continue } + // Nothing in it a node loads: no GGUF, no safetensors — pickles, + // which nodes refuse, or a format no engine here reads. + if e.Here && !loadable(m) { + e.Hidden++ + continue + } if e.Here && row.Verdict.Checked && !row.Verdict.Servable { e.Hidden++ continue @@ -295,3 +302,12 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) { } } } + +// loadable reports whether a listed repository has weights a node loads: +// GGUF or safetensors, by the registry's summary of them or its tags. +func loadable(m hf.Model) bool { + if m.GGUF != nil || m.Safetensors != nil { + return true + } + return slices.Contains(m.Tags, "gguf") || slices.Contains(m.Tags, "safetensors") +} diff --git a/internal/web/mirror_explore_test.go b/internal/web/mirror_explore_test.go index 65aa6e3a..dc026432 100644 --- a/internal/web/mirror_explore_test.go +++ b/internal/web/mirror_explore_test.go @@ -247,3 +247,52 @@ func TestAdaptersAreListedForAModelAndToldFromModelsByTheirFiles(t *testing.T) { t.Error("the merged file is not marked as a model") } } + +func TestARepositoryWithNothingANodeLoadsIsNotListed(t *testing.T) { + // Pickles only, or ONNX: a node refuses the first and no engine here + // reads the second, so the row would only invite a fetch that fails. + now := time.Now() + pickles := hf.Model{ID: "someone/old-pytorch", CreatedAt: now, LastModified: now, Tags: []string{"pytorch"}} + st := hf.Model{ID: "Qwen/Qwen3-8B", CreatedAt: now, LastModified: now, Tags: []string{"safetensors"}} + reg := &fakeRegistry{results: []hf.Model{pickles, st, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8.2e9, now)}} + mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}) + + body := mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String() + if strings.Contains(body, "someone/old-pytorch") { + t.Error("a repository with no GGUF and no safetensors is listed") + } + if !strings.Contains(body, "Qwen/Qwen3-8B<") && !strings.Contains(body, "Qwen/Qwen3-8B\"") || !strings.Contains(body, "Qwen/Qwen3-8B-GGUF") { + t.Errorf("the loadable repositories should stay: %s", body) + } + if !strings.Contains(body, "neither GGUF nor safetensors") { + t.Error("the page should say why one was hidden") + } + // Unticked, everything is shown. + body = mirrorGet(t, mux, "/admin/mirror?x=1&view=downloads&task=any").Body.String() + if !strings.Contains(body, "someone/old-pytorch") { + t.Error("with the box unticked the repository should be listed") + } +} + +func TestARepositorysPickleFilesAreNotOffered(t *testing.T) { + reg := &fakeRegistry{repo: hf.Repo{ + ID: "a/Both", Revision: "abc123def456", + Files: []hf.File{{Path: "model.safetensors", Bytes: 10}, {Path: "pytorch_model.bin", Bytes: 10}, + {Path: "config.json", Bytes: 1}}, + }} + mux := mirrorPageWith(t, &fakeMirror{}, reg, true) + body := mirrorGet(t, mux, "/admin/mirror?repo=a/Both").Body.String() + if strings.Contains(body, "pytorch_model.bin") { + t.Error("the pickle file is offered") + } + if !strings.Contains(body, "model.safetensors") || !strings.Contains(body, "1 file is not shown") { + t.Errorf("the safetensors should be offered and the pickle counted: %s", body) + } + + reg.repo = hf.Repo{ID: "a/OnlyPickle", Revision: "abc123def456", + Files: []hf.File{{Path: "model.ckpt", Bytes: 10}}} + body = html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=a/OnlyPickle").Body.String()) + if strings.Contains(body, "/resolve/abc123def456/model.ckpt") || !strings.Contains(body, "only in a pickle format") { + t.Errorf("a repository of pickles only should say so and offer nothing: %s", body) + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 37b3980c..58ac2e52 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -489,10 +489,11 @@ document.querySelector("table.explore").addEventListener("submit", async functio {{end}} +{{if .PickleHidden}}

{{tn "card.pickle_hidden" .PickleHidden}}

{{end}}

{{t "card.weights_note"}}

{{th "card.pinned_note" .Repo.ShortRevision}}

{{else}} -

{{t "card.no_weights" .Repo.ID}}

+

{{if .PickleHidden}}{{tn "card.only_pickle" .PickleHidden .Repo.ID}}{{else}}{{t "card.no_weights" .Repo.ID}}{{end}}

{{end}} {{end}}