From a588646b7c83473d1898ea0e3ecd4b510e7b7875 Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Sun, 27 Sep 2026 15:32:27 +0200
Subject: [PATCH] Fetch a sharded model in one action, not one shard at a time
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The explorer listed every weight file with its own button. For a repository of
alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded
model it is fifteen clicks to fetch one thing, with nothing stopping you from
losing count and leaving a version with fourteen of its parts.
Files are grouped into sets now. A run of "-00002-of-00004" is one row and one
button; everything else stays its own row.
Not "fetch all the weights", which would be wrong half the time: two safetensors
files are halves of one thing and useless apart, two GGUF files are usually
alternatives and fetching both spends gigabytes on a copy nobody asked for. The
set is the unit that is whole.
A repository can hold both the split and the joined copy of the same
quantisation — the real Qwen coder repos do — so a shard set keeps the -of-
notation in its name rather than collapsing onto the name of the whole file
beside it.
A set publishes no single checksum. Several files have several, and showing one
of them invites comparing it against the wrong thing.
The form posts every source in the set, and a failure part way says how many
already started: a set that fails on its fourth shard has three arriving, and
reading that as "nothing happened" sends somebody to fetch them all again.
Co-Authored-By: Claude Opus 5 (1M context)
---
internal/hf/hf.go | 64 ++++++++++++++++
internal/hf/hf_test.go | 51 +++++++++++++
internal/web/admin_mirror.go | 72 ++++++++++++------
internal/web/admin_mirror_test.go | 95 ++++++++++++++++++++++++
internal/web/templates/admin_mirror.html | 26 ++++---
5 files changed, 277 insertions(+), 31 deletions(-)
diff --git a/internal/hf/hf.go b/internal/hf/hf.go
index 775fac2..cdfa846 100644
--- a/internal/hf/hf.go
+++ b/internal/hf/hf.go
@@ -17,6 +17,7 @@ import (
"io"
"net/http"
"net/url"
+ "regexp"
"strings"
"time"
)
@@ -257,3 +258,66 @@ func (c *Client) get(ctx context.Context, path string, out any) error {
}
return nil
}
+
+// shardPattern matches the "-00002-of-00004" a split file carries before its
+// extension. Five digits in practice, but not required: the count is what
+// matters and a repository that used three would otherwise list every part as
+// a separate model.
+var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
+
+// FileSet is one thing worth fetching: a file, or every shard of one.
+//
+// The distinction the page turns on. Two GGUF files in a repository are usually
+// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
+// halves of the same weights and useless apart. Fetching "all the weights" is
+// right for the second and wasteful for the first, so neither is offered:
+// a set is offered, and a set is whole.
+type FileSet struct {
+ // Name is what to call it. For shards it keeps the -of- notation, because
+ // a repository can hold both the split and the joined copy of the same
+ // quantisation and they must not read as one entry.
+ Name string
+ Files []File
+ Bytes int64
+ Shards int
+}
+
+// SHA256 is the checksum when there is one to give.
+//
+// Empty for a shard set: several files have several checksums and no single
+// value for the set, and showing one of them invites comparing it against the
+// wrong thing.
+func (s FileSet) SHA256() string {
+ if len(s.Files) == 1 {
+ return s.Files[0].SHA256
+ }
+ return ""
+}
+
+// Group collects shards into sets, leaving every other file on its own.
+//
+// Order is preserved from the listing, so a repository that sorts its files
+// sensibly still reads sensibly here.
+func Group(files []File) []FileSet {
+ var out []FileSet
+ at := map[string]int{}
+ for _, f := range files {
+ m := shardPattern.FindStringSubmatch(f.Path)
+ if m == nil {
+ out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
+ continue
+ }
+ prefix, total, ext := m[1], m[3], m[4]
+ name := prefix + "-*-of-" + total + ext
+ i, ok := at[name]
+ if !ok {
+ at[name] = len(out)
+ out = append(out, FileSet{Name: name})
+ i = at[name]
+ }
+ out[i].Files = append(out[i].Files, f)
+ out[i].Bytes += f.Bytes
+ out[i].Shards = len(out[i].Files)
+ }
+ return out
+}
diff --git a/internal/hf/hf_test.go b/internal/hf/hf_test.go
index fd68845..8a8c904 100644
--- a/internal/hf/hf_test.go
+++ b/internal/hf/hf_test.go
@@ -136,3 +136,54 @@ func TestWeightsAreToldApartFromDocumentation(t *testing.T) {
}
}
}
+
+func TestShardsBecomeOneSetAndAlternativesStaySeparate(t *testing.T) {
+ // Straight from a real repository, which holds both the split and the
+ // joined copy of the same quantisation.
+ sets := Group([]File{
+ {Path: "qwen-fp16-00001-of-00004.gguf", Bytes: 1},
+ {Path: "qwen-fp16-00002-of-00004.gguf", Bytes: 1},
+ {Path: "qwen-fp16-00003-of-00004.gguf", Bytes: 1},
+ {Path: "qwen-fp16-00004-of-00004.gguf", Bytes: 1},
+ {Path: "qwen-fp16.gguf", Bytes: 4},
+ {Path: "qwen-q4_k_m.gguf", Bytes: 2},
+ })
+
+ if len(sets) != 3 {
+ t.Fatalf("expected the shards as one and the two whole files apart, got %d: %+v",
+ len(sets), sets)
+ }
+ if sets[0].Shards != 4 || sets[0].Bytes != 4 {
+ t.Errorf("the shard set should total its parts, got %+v", sets[0])
+ }
+ // Two GGUFs in a repository are alternatives: fetching both is wasting
+ // gigabytes on a copy nobody asked for.
+ if sets[1].Shards != 0 || sets[2].Shards != 0 {
+ t.Errorf("whole files are not shard sets: %+v %+v", sets[1], sets[2])
+ }
+}
+
+func TestASplitAndAJoinedCopyDoNotReadAsOneEntry(t *testing.T) {
+ sets := Group([]File{
+ {Path: "m-q4-00001-of-00002.gguf"}, {Path: "m-q4-00002-of-00002.gguf"},
+ {Path: "m-q4.gguf"},
+ })
+ if sets[0].Name == sets[1].Name {
+ t.Errorf("the split and the joined copy collided on %q", sets[0].Name)
+ }
+ if !strings.Contains(sets[0].Name, "-of-00002") {
+ t.Errorf("a shard set should keep the notation; got %q", sets[0].Name)
+ }
+}
+
+func TestAShardSetPublishesNoSingleChecksum(t *testing.T) {
+ sets := Group([]File{
+ {Path: "m-00001-of-00002.safetensors", SHA256: "aaa"},
+ {Path: "m-00002-of-00002.safetensors", SHA256: "bbb"},
+ })
+ // Several files have several checksums and no single value for the set;
+ // showing one invites comparing it against the wrong thing.
+ if got := sets[0].SHA256(); got != "" {
+ t.Errorf("expected no checksum for a set, got %q", got)
+ }
+}
diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go
index b4a6fea..835a079 100644
--- a/internal/web/admin_mirror.go
+++ b/internal/web/admin_mirror.go
@@ -3,6 +3,7 @@ package web
import (
"context"
"errors"
+ "fmt"
"net/http"
"sort"
"strconv"
@@ -36,10 +37,13 @@ type Registry interface {
Repo(ctx context.Context, id string) (hf.Repo, error)
}
-// repoFile is one file of a browsed repository, ready to be mirrored.
-type repoFile struct {
- hf.File
- Source string
+// repoSet is one thing worth fetching: a file, or every shard of one.
+type repoSet struct {
+ hf.FileSet
+ // Sources are every file in the set, in the order the registry listed them.
+ // A shard set goes as one action — a fifteen-shard model is not fifteen
+ // decisions, it is one.
+ Sources []repoSource
// Model and Version are what the fetch form starts with. Editable, because
// the name a catalog entry uses is the operator's to choose and the
// registry's idea of it is only a good guess.
@@ -47,6 +51,8 @@ type repoFile struct {
Version string
}
+type repoSource struct{ URL, Path string }
+
// mirrorRow is one model version as the page shows it.
type mirrorRow struct {
mirror.Status
@@ -79,7 +85,7 @@ type adminMirrorData struct {
Query string
Results []hf.Model
Repo *hf.Repo
- RepoFiles []repoFile
+ RepoSets []repoSet
RegistryErr string
Message string
@@ -112,23 +118,39 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re
mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
- req := mirror.FetchRequest{
- Source: strings.TrimSpace(r.FormValue("source")),
- Model: strings.TrimSpace(r.FormValue("model")),
- Version: strings.TrimSpace(r.FormValue("version")),
- File: strings.TrimSpace(r.FormValue("file")),
- }
- if req.Source == "" {
+ // One form posts one source or fifteen. A sharded model is one
+ // decision, not one per shard, and the mirror fetches each file
+ // independently anyway.
+ sources, files := r.Form["source"], r.Form["file"]
+ if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" {
return "", errors.New("a source URL is required")
}
- got, err := m.Fetch(ctx, req)
- if err != nil {
- return "", err
+ model := strings.TrimSpace(r.FormValue("model"))
+ version := strings.TrimSpace(r.FormValue("version"))
+
+ var started int
+ for i, src := range sources {
+ src = strings.TrimSpace(src)
+ if src == "" {
+ continue
+ }
+ req := mirror.FetchRequest{Source: src, Model: model, Version: version}
+ if i < len(files) {
+ req.File = strings.TrimSpace(files[i])
+ }
+ if _, err := m.Fetch(ctx, req); err != nil {
+ // Reported with what already went, so a set that fails on
+ // its fourth shard does not read as though nothing started.
+ return "", fmt.Errorf("%d of %d started, then %s: %w",
+ started, len(sources), mirror.LastPathElement(src), err)
+ }
+ started++
}
// Named as started, not as done. The mirror answered 202 and the
// bytes are still moving; a message saying "added" would invite
// somebody to point the catalog at a partial file.
- return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil
+ return fmt.Sprintf("started downloading %s/%s (%s)",
+ model, version, plural(started, "file", "files")), nil
}))
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
@@ -278,6 +300,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
return
}
d.Repo = &got
+ var weights []hf.File
for _, f := range got.Files {
if !f.IsWeights() {
// Thirty files of which four are worth gigabytes. The rest are
@@ -285,15 +308,22 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
// somebody came for.
continue
}
- d.RepoFiles = append(d.RepoFiles, repoFile{
- File: f,
- Source: got.DownloadURL("", f.Path),
- Model: defaultModelName(got.ID),
+ weights = append(weights, f)
+ }
+ for _, set := range hf.Group(weights) {
+ rs := repoSet{
+ FileSet: set,
+ Model: defaultModelName(got.ID),
// The revision, not the date: two fetches of the same commit
// land in the same place, and a repository re-uploaded under
// the same branch lands somewhere else.
Version: got.ShortRevision(),
- })
+ }
+ for _, f := range set.Files {
+ rs.Sources = append(rs.Sources,
+ repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
+ }
+ d.RepoSets = append(d.RepoSets, rs)
}
return
}
diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go
index 4b91cc9..1327973 100644
--- a/internal/web/admin_mirror_test.go
+++ b/internal/web/admin_mirror_test.go
@@ -52,6 +52,7 @@ type fakeMirror struct {
models []mirror.Status
listErr error
fetched mirror.FetchRequest
+ all []mirror.FetchRequest
fetchErr error
removed [2]string
removeErr error
@@ -68,6 +69,7 @@ func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
f.fetched = r
+ f.all = append(f.all, r)
if f.fetchErr != nil {
return mirror.FetchResponse{}, f.fetchErr
}
@@ -435,3 +437,96 @@ func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) {
t.Error("fetching by hand must still be there")
}
}
+
+func shardedRepo() *fakeRegistry {
+ return &fakeRegistry{repo: hf.Repo{
+ ID: "Qwen/Qwen2.5-7B-Instruct", Revision: "abc123def456",
+ Files: []hf.File{
+ {Path: "model-00001-of-00004.safetensors", Bytes: 4_000_000_000},
+ {Path: "model-00002-of-00004.safetensors", Bytes: 4_000_000_000},
+ {Path: "model-00003-of-00004.safetensors", Bytes: 4_000_000_000},
+ {Path: "model-00004-of-00004.safetensors", Bytes: 3_000_000_000},
+ {Path: "README.md", Bytes: 900},
+ },
+ }}
+}
+
+func TestAShardedModelIsOneRowAndOneButton(t *testing.T) {
+ body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, shardedRepo(), true),
+ "/admin/mirror?repo=Qwen/Qwen2.5-7B-Instruct").Body.String()
+
+ // Fifteen shards is one decision, not fifteen. Four here, and one form.
+ if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 {
+ // One for the set, one for the manual form at the bottom.
+ t.Errorf("expected the four shards as a single form, found %d forms", n)
+ }
+ if !strings.Contains(body, "4 shards, fetched together") {
+ t.Errorf("the row should say it is a set; got %s", body)
+ }
+ // Summed, so the operator sees what the disk is in for.
+ if !strings.Contains(body, "14.0 GiB") {
+ t.Error("the set should total its shards")
+ }
+}
+
+func TestFetchingASetStartsEveryShard(t *testing.T) {
+ m := &fakeMirror{}
+ form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"abc123de"}}
+ for i, p := range []string{"model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"} {
+ form.Add("source", "https://hf/resolve/abc123def456/"+p)
+ form.Add("file", p)
+ _ = i
+ }
+ body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
+ "/admin/mirror/fetch", form).Body.String()
+
+ if len(m.all) != 2 {
+ t.Fatalf("expected both shards to be started, got %d", len(m.all))
+ }
+ // Each shard keeps its own filename under one model and version, which is
+ // what makes them one version on disk.
+ if m.all[0].File == m.all[1].File {
+ t.Errorf("the shards collided on %q", m.all[0].File)
+ }
+ if m.all[0].Model != "qwen" || m.all[1].Version != "abc123de" {
+ t.Errorf("the set should share one model and version: %+v", m.all)
+ }
+ if !strings.Contains(body, "2 files") {
+ t.Errorf("the message should say how many started; got %s", body)
+ }
+}
+
+func TestASetThatFailsPartWaySaysHowFarItGot(t *testing.T) {
+ m := &fakeMirror{fetchErr: errors.New("the mirror is full")}
+ form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"v"}}
+ form.Add("source", "https://hf/a.safetensors")
+ form.Add("file", "a.safetensors")
+
+ body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
+ "/admin/mirror/fetch", form).Body.String()
+
+ // "0 of 1 started, then …" rather than a bare error: a set that fails on
+ // its fourth shard has three arriving, and reading it as nothing started
+ // would send somebody to fetch them all again.
+ if !strings.Contains(body, "0 of 1 started") {
+ t.Errorf("the message should say how far it got; got %s", body)
+ }
+}
+
+func TestAlternativeQuantisationsStaySeparateRows(t *testing.T) {
+ reg := &fakeRegistry{repo: hf.Repo{
+ ID: "Qwen/X", Revision: "abc",
+ Files: []hf.File{
+ {Path: "m-q4_k_m.gguf", Bytes: 1},
+ {Path: "m-q8_0.gguf", Bytes: 2},
+ },
+ }}
+ body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
+ "/admin/mirror?repo=Qwen/X").Body.String()
+
+ // Two GGUFs are alternatives, not halves. Offering "fetch all" here would
+ // spend gigabytes on a copy nobody asked for.
+ if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 3 {
+ t.Errorf("expected a form each plus the manual one, found %d", n)
+ }
+}
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index e435d84..c827653 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -120,18 +120,20 @@ directory, or give the mirror a token.
{{if .Repo}}
{{.Repo.ID}} at {{.Repo.ShortRevision}}
-{{if .RepoFiles}}
+{{if .RepoSets}}
-| File | Size | Published checksum | |
-{{range .RepoFiles}}
- {{.Path}} |
+
| What | Size | Published checksum | |
+{{range .RepoSets}}
+ {{.Name}}
+ {{if .Shards}}{{.Shards}} shards, fetched together{{end}} |
{{bytes .Bytes}} |
- {{if .SHA256}}{{slice .SHA256 0 12}}{{else}}—{{end}} |
+ {{with .SHA256}}{{slice . 0 12}}{{else}}—{{end}} |
|
{{end}}
Only the weights are listed; a repository is mostly a README and
-a tokenizer. The source URL is pinned to {{.Repo.ShortRevision}}
-rather than to a branch, so what arrives is what is listed here even if the
-branch moves in between — and the checksum above is what the registry publishes,
-to compare with what the mirror computes once it has the file.
+a tokenizer. Shards of one file are one row and one button — a fifteen-shard
+model is one decision, not fifteen. Two separate files are two rows because they
+are usually alternatives: one quantisation or the other, not both.
+The source URLs are pinned to
+{{.Repo.ShortRevision}} rather than to a branch, so what arrives is
+what is listed here even if the branch moves in between — and a single file's
+checksum above is what the registry publishes, to compare with what the mirror
+computes once it has it.
{{else}}
No weight files in {{.Repo.ID}}.
{{end}}