Fetch a sharded model in one action, not one shard at a time
CI / test (push) Successful in 1m39s

The explorer listed every weight file with its own button. For a repository of
alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded
model it is fifteen clicks to fetch one thing, with nothing stopping you from
losing count and leaving a version with fourteen of its parts.

Files are grouped into sets now. A run of "-00002-of-00004" is one row and one
button; everything else stays its own row.

Not "fetch all the weights", which would be wrong half the time: two safetensors
files are halves of one thing and useless apart, two GGUF files are usually
alternatives and fetching both spends gigabytes on a copy nobody asked for. The
set is the unit that is whole.

A repository can hold both the split and the joined copy of the same
quantisation — the real Qwen coder repos do — so a shard set keeps the -of-
notation in its name rather than collapsing onto the name of the whole file
beside it.

A set publishes no single checksum. Several files have several, and showing one
of them invites comparing it against the wrong thing.

The form posts every source in the set, and a failure part way says how many
already started: a set that fails on its fourth shard has three arriving, and
reading that as "nothing happened" sends somebody to fetch them all again.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 15:32:27 +02:00
1 parent ed530133b6
commit a588646b7c
5 files changed
+277 -31

No files matched your search

+51 -21
View File
@@ -3,6 +3,7 @@ package web
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
"strconv"
@@ -36,10 +37,13 @@ type Registry interface {
Repo(ctx context.Context, id string) (hf.Repo, error)
}
// repoFile is one file of a browsed repository, ready to be mirrored.
type repoFile struct {
hf.File
Source string
// repoSet is one thing worth fetching: a file, or every shard of one.
type repoSet struct {
hf.FileSet
// Sources are every file in the set, in the order the registry listed them.
// A shard set goes as one action — a fifteen-shard model is not fifteen
// decisions, it is one.
Sources []repoSource
// Model and Version are what the fetch form starts with. Editable, because
// the name a catalog entry uses is the operator's to choose and the
// registry's idea of it is only a good guess.
@@ -47,6 +51,8 @@ type repoFile struct {
Version string
}
type repoSource struct{ URL, Path string }
// mirrorRow is one model version as the page shows it.
type mirrorRow struct {
mirror.Status
@@ -79,7 +85,7 @@ type adminMirrorData struct {
Query string
Results []hf.Model
Repo *hf.Repo
RepoFiles []repoFile
RepoSets []repoSet
RegistryErr string
Message string
@@ -112,23 +118,39 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re
mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
req := mirror.FetchRequest{
Source: strings.TrimSpace(r.FormValue("source")),
Model: strings.TrimSpace(r.FormValue("model")),
Version: strings.TrimSpace(r.FormValue("version")),
File: strings.TrimSpace(r.FormValue("file")),
}
if req.Source == "" {
// One form posts one source or fifteen. A sharded model is one
// decision, not one per shard, and the mirror fetches each file
// independently anyway.
sources, files := r.Form["source"], r.Form["file"]
if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" {
return "", errors.New("a source URL is required")
}
got, err := m.Fetch(ctx, req)
if err != nil {
return "", err
model := strings.TrimSpace(r.FormValue("model"))
version := strings.TrimSpace(r.FormValue("version"))
var started int
for i, src := range sources {
src = strings.TrimSpace(src)
if src == "" {
continue
}
req := mirror.FetchRequest{Source: src, Model: model, Version: version}
if i < len(files) {
req.File = strings.TrimSpace(files[i])
}
if _, err := m.Fetch(ctx, req); err != nil {
// Reported with what already went, so a set that fails on
// its fourth shard does not read as though nothing started.
return "", fmt.Errorf("%d of %d started, then %s: %w",
started, len(sources), mirror.LastPathElement(src), err)
}
started++
}
// Named as started, not as done. The mirror answered 202 and the
// bytes are still moving; a message saying "added" would invite
// somebody to point the catalog at a partial file.
return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil
return fmt.Sprintf("started downloading %s/%s (%s)",
model, version, plural(started, "file", "files")), nil
}))
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
@@ -278,6 +300,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
return
}
d.Repo = &got
var weights []hf.File
for _, f := range got.Files {
if !f.IsWeights() {
// Thirty files of which four are worth gigabytes. The rest are
@@ -285,15 +308,22 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
// somebody came for.
continue
}
d.RepoFiles = append(d.RepoFiles, repoFile{
File: f,
Source: got.DownloadURL("", f.Path),
Model: defaultModelName(got.ID),
weights = append(weights, f)
}
for _, set := range hf.Group(weights) {
rs := repoSet{
FileSet: set,
Model: defaultModelName(got.ID),
// The revision, not the date: two fetches of the same commit
// land in the same place, and a repository re-uploaded under
// the same branch lands somewhere else.
Version: got.ShortRevision(),
})
}
for _, f := range set.Files {
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
}
d.RepoSets = append(d.RepoSets, rs)
}
return
}