diff --git a/BACKLOG.md b/BACKLOG.md
index e7d671d9..c3134dc5 100644
--- a/BACKLOG.md
+++ b/BACKLOG.md
@@ -1800,8 +1800,9 @@ and never overwritten. What did not: everything below.
how the malicious models found on Hugging Face work. The node refuses such a
source, or a mirror set holding one, before fetching anything; a model being
added or given such a source is refused; vLLM and SGLang downloading a model
-by its id are given `--load-format safetensors`; the mirror page says so on
-the file. `allow_pickle: true` on a model lets it through, on purpose. On the
+by its id are given `--load-format safetensors`. The explorer hides a repository with neither GGUF nor safetensors
+(under "runs here", ticked by default) and does not offer a repository's
+pickle files. `allow_pickle: true` on a model lets it through, on purpose. On the
PoC, `triposr` (`hf:stabilityai/TripoSR/model.ckpt`) is the one model that
needs it.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a5a87d3b..5bf6cb70 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,7 +45,10 @@ APIs change between minors without a deprecation period.
or a mirror version with one in it, before fetching it; adding such a
model is refused; vLLM and SGLang downloading a model by its id read its
safetensors only; the mirror page marks the file. `allow_pickle: true`
- on a model lets it through on purpose. BACKLOG M lists what is next:
+ on a model lets it through on purpose. The mirror's explorer no longer
+ lists a repository with nothing a node loads (no GGUF, no safetensors)
+ while "runs here" is ticked, and a repository's pickle files are not
+ offered, only counted. BACKLOG M lists what is next:
checksums against Hugging Face everywhere, its own scan, trusted
publishers, engines without the internet.
diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go
index ae7175ec..88c19dbe 100644
--- a/internal/web/admin_mirror.go
+++ b/internal/web/admin_mirror.go
@@ -14,7 +14,7 @@ import (
"sync"
"time"
- "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
+ weightspkg "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
@@ -175,10 +175,13 @@ type adminMirrorData struct {
// Card is the open repository's model card, rendered.
Card template.HTML
// RepoURL is the open repository on the registry's own site.
- RepoURL string
- Repo *hf.Repo
- RepoSets []repoSet
- RegistryErr string
+ RepoURL string
+ Repo *hf.Repo
+ RepoSets []repoSet
+ // PickleHidden is how many of the open repository's weights files were
+ // left out for being in a pickle format, which nodes refuse.
+ PickleHidden int
+ RegistryErr string
Message string
Error string
@@ -789,6 +792,12 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
d.Card = renderCard(got.Card, d.RepoURL+"/blob/"+revOrMain(got.Revision)+"/")
var weights []hf.File
for _, f := range got.Files {
+ // Pickles run code when loaded and a node refuses them unless a
+ // model says allow_pickle: not offered, only counted.
+ if weightspkg.Pickled(f.Path) {
+ d.PickleHidden++
+ continue
+ }
if !f.IsWeights() {
// Thirty files of which four are worth gigabytes. The rest are
// a README and a tokenizer, and listing them buries the ones
@@ -877,11 +886,6 @@ func defaultModelName(repoID string) string { return hf.ModelName(repoID) }
// verdictFor judges a registry file set by the repository's architecture. A
// vision projector is its own architecture whatever the repository says.
func (p *AccountPages) verdictFor(l *Lang, arch, name string) gguf.Verdict {
- // Not refused here — the mirror keeps what it is given — but a node
- // refuses to fetch it for a model without allow_pickle.
- if weights.Pickled(name) {
- return gguf.Verdict{Checked: true, Reason: l.T("card.pickle")}
- }
if strings.Contains(strings.ToLower(name), "mmproj") {
return gguf.LlamaCpp(p.llamaVersion, "clip")
}
diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go
index 9f1cf638..ff8aa567 100644
--- a/internal/web/admin_mirror_test.go
+++ b/internal/web/admin_mirror_test.go
@@ -364,7 +364,7 @@ func (f *fakeRegistry) Repo(_ context.Context, id string) (hf.Repo, error) {
func TestSearchingTheRegistryListsRepositories(t *testing.T) {
reg := &fakeRegistry{results: []hf.Model{
- {ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation"},
+ {ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation", Tags: []string{"gguf"}},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?q=qwen+coder").Body.String()
diff --git a/internal/web/i18n_mirror_test.go b/internal/web/i18n_mirror_test.go
index 12072b82..87b9b04f 100644
--- a/internal/web/i18n_mirror_test.go
+++ b/internal/web/i18n_mirror_test.go
@@ -49,7 +49,7 @@ func TestTheMirrorPageSpeaksTurkishWhenAsked(t *testing.T) {
{Model: "llama", Version: "v1", File: "l.gguf", State: mirror.Fetching,
BytesDone: 1 << 20, BytesTotal: 4 << 20},
}}
- reg := &fakeRegistry{results: []hf.Model{{ID: "Qwen/Qwen3-8B-GGUF", Downloads: 1247,
+ reg := &fakeRegistry{results: []hf.Model{{ID: "Qwen/Qwen3-8B-GGUF", Downloads: 1247, Tags: []string{"gguf"},
CreatedAt: time.Now().Add(-3 * time.Hour)}}}
mux := mirrorPageWith(t, m, reg, true)
diff --git a/internal/web/locales/en/mirror.json b/internal/web/locales/en/mirror.json
index dec737bf..9ecee295 100644
--- a/internal/web/locales/en/mirror.json
+++ b/internal/web/locales/en/mirror.json
@@ -95,15 +95,15 @@
"explore.js.follow": "follow it under Held",
"explore.js.started": "Started",
"explore.js.unreachable": "Could not reach the gateway:",
- "explore.hidden.one": "%d more is hidden: their architecture is one the nodes' llama.cpp cannot load.",
- "explore.hidden.other": "%d more are hidden: their architecture is one the nodes' llama.cpp cannot load.",
+ "explore.hidden.one": "%d more is hidden: nothing in it these nodes can load — an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
+ "explore.hidden.other": "%d more are hidden: nothing in them these nodes can load — an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
"explore.untick": "Untick “only what these nodes can run” to see them.",
"explore.gated_note": "A gated repository needs its terms accepted with a Hugging Face account, and this browses anonymously — the files will list, and fetching one will fail at the mirror. Download it by hand and put it in the mirror directory, or give the mirror a token.",
- "explore.empty.query_hidden.one": "Nothing matched “%[2]s” that these nodes can run — %[1]d result was hidden for an architecture llama.cpp cannot load.",
- "explore.empty.query_hidden.other": "Nothing matched “%[2]s” that these nodes can run — %[1]d results were hidden for an architecture llama.cpp cannot load.",
+ "explore.empty.query_hidden.one": "Nothing matched “%[2]s” that these nodes can run — %[1]d result was hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
+ "explore.empty.query_hidden.other": "Nothing matched “%[2]s” that these nodes can run — %[1]d results were hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
"explore.empty.query": "Nothing matched “%s”.",
- "explore.empty.hidden.one": "Nothing to list that these nodes can run — %d result was hidden for an architecture llama.cpp cannot load.",
- "explore.empty.hidden.other": "Nothing to list that these nodes can run — %d results were hidden for an architecture llama.cpp cannot load.",
+ "explore.empty.hidden.one": "Nothing to list that these nodes can run — %d result was hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
+ "explore.empty.hidden.other": "Nothing to list that these nodes can run — %d results were hidden: an architecture llama.cpp cannot load, or weights in neither GGUF nor safetensors.",
"explore.empty.nothing": "Nothing to list.",
"mirror.fetch.intro": "By hand, for anything the search does not reach.",
@@ -137,7 +137,10 @@
"card.model": "model",
"card.cannot_serve": "cannot be served",
"card.imatrix": "it is an importance matrix for quantising, not a model",
- "card.pickle": "it is in a pickle format, which runs code when it is loaded; nodes refuse it unless the model says allow_pickle — take the safetensors or GGUF",
+ "card.pickle_hidden.one": "%d file is not shown: it is in a pickle format, which runs code when it is loaded, and nodes refuse it.",
+ "card.pickle_hidden.other": "%d files are not shown: they are in a pickle format, which runs code when it is loaded, and nodes refuse them.",
+ "card.only_pickle.one": "%[2]s has weights only in a pickle format (%[1]d file), which runs code when it is loaded; nodes refuse it. Look for a safetensors or GGUF version of the model.",
+ "card.only_pickle.other": "%[2]s has weights only in a pickle format (%[1]d files), which runs code when it is loaded; nodes refuse them. Look for a safetensors or GGUF version of the model.",
"card.fetch": "Fetch",
"card.weights_note": "Only the weights are listed; a repository is mostly a README and a tokenizer. A safetensors set is fetched with the files vLLM reads beside it — config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard model is one decision, not fifteen. Two separate files are two rows because they are usually alternatives: one quantisation or the other, not both.",
"card.pinned_note": "The source URLs are pinned to %s rather than to a branch, so what arrives is what is listed here even if the branch moves in between — and a single file's checksum above is what the registry publishes, to compare with what the mirror computes once it has it.",
diff --git a/internal/web/locales/tr/mirror.json b/internal/web/locales/tr/mirror.json
index a065c07a..ce6dae3d 100644
--- a/internal/web/locales/tr/mirror.json
+++ b/internal/web/locales/tr/mirror.json
@@ -93,12 +93,12 @@
"explore.js.follow": "Aynadakiler sekmesinden izleyin",
"explore.js.started": "Başladı",
"explore.js.unreachable": "Ağ geçidine ulaşılamadı:",
- "explore.hidden.other": "%d sonuç daha gizlendi: mimarileri düğümlerdeki llama.cpp'nin yükleyemediği bir mimari.",
+ "explore.hidden.other": "%d sonuç daha gizlendi: düğümlerin yükleyebileceği bir şey yok — llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.",
"explore.untick": "Onları görmek için “yalnızca bu düğümlerin çalıştırabildikleri” kutusunun işaretini kaldırın.",
"explore.gated_note": "Erişimi kısıtlı bir depo, koşullarının bir Hugging Face hesabıyla kabul edilmesini ister; buradaki tarama ise anonimdir — dosyalar listelenir, ancak birini indirmek aynada başarısız olur. Dosyayı elle indirip ayna dizinine koyun ya da aynaya bir token verin.",
- "explore.empty.query_hidden.other": "“%[2]s” ile eşleşen ve bu düğümlerin çalıştırabileceği bir şey yok — llama.cpp'nin yükleyemediği bir mimari nedeniyle %[1]d sonuç gizlendi.",
+ "explore.empty.query_hidden.other": "“%[2]s” ile eşleşen ve bu düğümlerin çalıştırabileceği bir şey yok — %[1]d sonuç gizlendi: llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.",
"explore.empty.query": "“%s” ile eşleşen bir şey yok.",
- "explore.empty.hidden.other": "Bu düğümlerin çalıştırabileceği listelenecek bir şey yok — llama.cpp'nin yükleyemediği bir mimari nedeniyle %d sonuç gizlendi.",
+ "explore.empty.hidden.other": "Bu düğümlerin çalıştırabileceği listelenecek bir şey yok — %d sonuç gizlendi: llama.cpp'nin yükleyemediği bir mimari, ya da ne GGUF ne safetensors olan ağırlıklar.",
"explore.empty.nothing": "Listelenecek bir şey yok.",
"mirror.fetch.intro": "Aramanın ulaşamadığı her şey için, elle.",
@@ -132,7 +132,8 @@
"card.model": "model",
"card.cannot_serve": "sunulamaz",
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
- "card.pickle": "pickle biçiminde; yüklenirken kod çalıştırır. Model allow_pickle demedikçe düğümler onu almaz — safetensors ya da GGUF olanı seçin",
+ "card.pickle_hidden.other": "%d dosya gösterilmiyor: pickle biçiminde, yüklenirken kod çalıştırır ve düğümler onu almaz.",
+ "card.only_pickle.other": "%[2]s deposunun ağırlıkları yalnızca pickle biçiminde (%[1]d dosya); yüklenirken kod çalıştırır, düğümler almaz. Modelin safetensors ya da GGUF sürümünü arayın.",
"card.fetch": "İndir",
"card.weights_note": "Yalnızca ağırlıklar listelenir; bir deponun çoğu bir README ve bir tokenizer'dır. Bir safetensors kümesi, vLLM'in yanında okuduğu dosyalarla — config.json, tokenizer, parça dizini — birlikte indirilir, asla kodla birlikte değil. Bir dosyanın parçaları tek satır ve tek düğmedir — on beş parçalı bir model on beş değil, tek bir karardır. Ayrı iki dosya iki satırdır, çünkü genellikle birbirinin seçeneğidir: ya bir nicemleme ya öteki, ikisi birden değil.",
"card.pinned_note": "Kaynak URL'ler bir dala değil %s sürümüne sabitlenmiştir; böylece dal arada değişse bile gelen, burada listelenendir — yukarıdaki tek dosya sağlama toplamı da model deposunun yayımladığıdır ve ayna dosyayı aldığında hesapladığıyla karşılaştırılabilir.",
diff --git a/internal/web/mirror_explore.go b/internal/web/mirror_explore.go
index 607ff2e1..496434b5 100644
--- a/internal/web/mirror_explore.go
+++ b/internal/web/mirror_explore.go
@@ -5,6 +5,7 @@ import (
"html/template"
"net/http"
"net/url"
+ "slices"
"strings"
"time"
@@ -286,6 +287,12 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
if e.Here && m.Params() > 0 && m.Params() < minParams {
continue
}
+ // Nothing in it a node loads: no GGUF, no safetensors — pickles,
+ // which nodes refuse, or a format no engine here reads.
+ if e.Here && !loadable(m) {
+ e.Hidden++
+ continue
+ }
if e.Here && row.Verdict.Checked && !row.Verdict.Servable {
e.Hidden++
continue
@@ -295,3 +302,12 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
}
}
}
+
+// loadable reports whether a listed repository has weights a node loads:
+// GGUF or safetensors, by the registry's summary of them or its tags.
+func loadable(m hf.Model) bool {
+ if m.GGUF != nil || m.Safetensors != nil {
+ return true
+ }
+ return slices.Contains(m.Tags, "gguf") || slices.Contains(m.Tags, "safetensors")
+}
diff --git a/internal/web/mirror_explore_test.go b/internal/web/mirror_explore_test.go
index 65aa6e3a..dc026432 100644
--- a/internal/web/mirror_explore_test.go
+++ b/internal/web/mirror_explore_test.go
@@ -247,3 +247,52 @@ func TestAdaptersAreListedForAModelAndToldFromModelsByTheirFiles(t *testing.T) {
t.Error("the merged file is not marked as a model")
}
}
+
+func TestARepositoryWithNothingANodeLoadsIsNotListed(t *testing.T) {
+ // Pickles only, or ONNX: a node refuses the first and no engine here
+ // reads the second, so the row would only invite a fetch that fails.
+ now := time.Now()
+ pickles := hf.Model{ID: "someone/old-pytorch", CreatedAt: now, LastModified: now, Tags: []string{"pytorch"}}
+ st := hf.Model{ID: "Qwen/Qwen3-8B", CreatedAt: now, LastModified: now, Tags: []string{"safetensors"}}
+ reg := &fakeRegistry{results: []hf.Model{pickles, st, ggufModel("Qwen/Qwen3-8B-GGUF", "qwen3", 8.2e9, now)}}
+ mux := mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg})
+
+ body := mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String()
+ if strings.Contains(body, "someone/old-pytorch") {
+ t.Error("a repository with no GGUF and no safetensors is listed")
+ }
+ if !strings.Contains(body, "Qwen/Qwen3-8B<") && !strings.Contains(body, "Qwen/Qwen3-8B\"") || !strings.Contains(body, "Qwen/Qwen3-8B-GGUF") {
+ t.Errorf("the loadable repositories should stay: %s", body)
+ }
+ if !strings.Contains(body, "neither GGUF nor safetensors") {
+ t.Error("the page should say why one was hidden")
+ }
+ // Unticked, everything is shown.
+ body = mirrorGet(t, mux, "/admin/mirror?x=1&view=downloads&task=any").Body.String()
+ if !strings.Contains(body, "someone/old-pytorch") {
+ t.Error("with the box unticked the repository should be listed")
+ }
+}
+
+func TestARepositorysPickleFilesAreNotOffered(t *testing.T) {
+ reg := &fakeRegistry{repo: hf.Repo{
+ ID: "a/Both", Revision: "abc123def456",
+ Files: []hf.File{{Path: "model.safetensors", Bytes: 10}, {Path: "pytorch_model.bin", Bytes: 10},
+ {Path: "config.json", Bytes: 1}},
+ }}
+ mux := mirrorPageWith(t, &fakeMirror{}, reg, true)
+ body := mirrorGet(t, mux, "/admin/mirror?repo=a/Both").Body.String()
+ if strings.Contains(body, "pytorch_model.bin") {
+ t.Error("the pickle file is offered")
+ }
+ if !strings.Contains(body, "model.safetensors") || !strings.Contains(body, "1 file is not shown") {
+ t.Errorf("the safetensors should be offered and the pickle counted: %s", body)
+ }
+
+ reg.repo = hf.Repo{ID: "a/OnlyPickle", Revision: "abc123def456",
+ Files: []hf.File{{Path: "model.ckpt", Bytes: 10}}}
+ body = html.UnescapeString(mirrorGet(t, mux, "/admin/mirror?repo=a/OnlyPickle").Body.String())
+ if strings.Contains(body, "/resolve/abc123def456/model.ckpt") || !strings.Contains(body, "only in a pickle format") {
+ t.Errorf("a repository of pickles only should say so and offer nothing: %s", body)
+ }
+}
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index 37b3980c..58ac2e52 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -489,10 +489,11 @@ document.querySelector("table.explore").addEventListener("submit", async functio
{{end}}
+{{if .PickleHidden}}
{{tn "card.pickle_hidden" .PickleHidden}}
{{end}}{{t "card.weights_note"}}
{{th "card.pinned_note" .Repo.ShortRevision}}
{{else}} -{{t "card.no_weights" .Repo.ID}}
+{{if .PickleHidden}}{{tn "card.only_pickle" .PickleHidden .Repo.ID}}{{else}}{{t "card.no_weights" .Repo.ID}}{{end}}
{{end}} {{end}}