diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 775fac2a..cdfa8465 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -17,6 +17,7 @@ import ( "io" "net/http" "net/url" + "regexp" "strings" "time" ) @@ -257,3 +258,66 @@ func (c *Client) get(ctx context.Context, path string, out any) error { } return nil } + +// shardPattern matches the "-00002-of-00004" a split file carries before its +// extension. Five digits in practice, but not required: the count is what +// matters and a repository that used three would otherwise list every part as +// a separate model. +var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`) + +// FileSet is one thing worth fetching: a file, or every shard of one. +// +// The distinction the page turns on. Two GGUF files in a repository are usually +// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are +// halves of the same weights and useless apart. Fetching "all the weights" is +// right for the second and wasteful for the first, so neither is offered: +// a set is offered, and a set is whole. +type FileSet struct { + // Name is what to call it. For shards it keeps the -of- notation, because + // a repository can hold both the split and the joined copy of the same + // quantisation and they must not read as one entry. + Name string + Files []File + Bytes int64 + Shards int +} + +// SHA256 is the checksum when there is one to give. +// +// Empty for a shard set: several files have several checksums and no single +// value for the set, and showing one of them invites comparing it against the +// wrong thing. +func (s FileSet) SHA256() string { + if len(s.Files) == 1 { + return s.Files[0].SHA256 + } + return "" +} + +// Group collects shards into sets, leaving every other file on its own. +// +// Order is preserved from the listing, so a repository that sorts its files +// sensibly still reads sensibly here. +func Group(files []File) []FileSet { + var out []FileSet + at := map[string]int{} + for _, f := range files { + m := shardPattern.FindStringSubmatch(f.Path) + if m == nil { + out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes}) + continue + } + prefix, total, ext := m[1], m[3], m[4] + name := prefix + "-*-of-" + total + ext + i, ok := at[name] + if !ok { + at[name] = len(out) + out = append(out, FileSet{Name: name}) + i = at[name] + } + out[i].Files = append(out[i].Files, f) + out[i].Bytes += f.Bytes + out[i].Shards = len(out[i].Files) + } + return out +} diff --git a/internal/hf/hf_test.go b/internal/hf/hf_test.go index fd688452..8a8c9042 100644 --- a/internal/hf/hf_test.go +++ b/internal/hf/hf_test.go @@ -136,3 +136,54 @@ func TestWeightsAreToldApartFromDocumentation(t *testing.T) { } } } + +func TestShardsBecomeOneSetAndAlternativesStaySeparate(t *testing.T) { + // Straight from a real repository, which holds both the split and the + // joined copy of the same quantisation. + sets := Group([]File{ + {Path: "qwen-fp16-00001-of-00004.gguf", Bytes: 1}, + {Path: "qwen-fp16-00002-of-00004.gguf", Bytes: 1}, + {Path: "qwen-fp16-00003-of-00004.gguf", Bytes: 1}, + {Path: "qwen-fp16-00004-of-00004.gguf", Bytes: 1}, + {Path: "qwen-fp16.gguf", Bytes: 4}, + {Path: "qwen-q4_k_m.gguf", Bytes: 2}, + }) + + if len(sets) != 3 { + t.Fatalf("expected the shards as one and the two whole files apart, got %d: %+v", + len(sets), sets) + } + if sets[0].Shards != 4 || sets[0].Bytes != 4 { + t.Errorf("the shard set should total its parts, got %+v", sets[0]) + } + // Two GGUFs in a repository are alternatives: fetching both is wasting + // gigabytes on a copy nobody asked for. + if sets[1].Shards != 0 || sets[2].Shards != 0 { + t.Errorf("whole files are not shard sets: %+v %+v", sets[1], sets[2]) + } +} + +func TestASplitAndAJoinedCopyDoNotReadAsOneEntry(t *testing.T) { + sets := Group([]File{ + {Path: "m-q4-00001-of-00002.gguf"}, {Path: "m-q4-00002-of-00002.gguf"}, + {Path: "m-q4.gguf"}, + }) + if sets[0].Name == sets[1].Name { + t.Errorf("the split and the joined copy collided on %q", sets[0].Name) + } + if !strings.Contains(sets[0].Name, "-of-00002") { + t.Errorf("a shard set should keep the notation; got %q", sets[0].Name) + } +} + +func TestAShardSetPublishesNoSingleChecksum(t *testing.T) { + sets := Group([]File{ + {Path: "m-00001-of-00002.safetensors", SHA256: "aaa"}, + {Path: "m-00002-of-00002.safetensors", SHA256: "bbb"}, + }) + // Several files have several checksums and no single value for the set; + // showing one invites comparing it against the wrong thing. + if got := sets[0].SHA256(); got != "" { + t.Errorf("expected no checksum for a set, got %q", got) + } +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index b4a6fea5..835a0792 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -3,6 +3,7 @@ package web import ( "context" "errors" + "fmt" "net/http" "sort" "strconv" @@ -36,10 +37,13 @@ type Registry interface { Repo(ctx context.Context, id string) (hf.Repo, error) } -// repoFile is one file of a browsed repository, ready to be mirrored. -type repoFile struct { - hf.File - Source string +// repoSet is one thing worth fetching: a file, or every shard of one. +type repoSet struct { + hf.FileSet + // Sources are every file in the set, in the order the registry listed them. + // A shard set goes as one action — a fifteen-shard model is not fifteen + // decisions, it is one. + Sources []repoSource // Model and Version are what the fetch form starts with. Editable, because // the name a catalog entry uses is the operator's to choose and the // registry's idea of it is only a good guess. @@ -47,6 +51,8 @@ type repoFile struct { Version string } +type repoSource struct{ URL, Path string } + // mirrorRow is one model version as the page shows it. type mirrorRow struct { mirror.Status @@ -79,7 +85,7 @@ type adminMirrorData struct { Query string Results []hf.Model Repo *hf.Repo - RepoFiles []repoFile + RepoSets []repoSet RegistryErr string Message string @@ -112,23 +118,39 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { - req := mirror.FetchRequest{ - Source: strings.TrimSpace(r.FormValue("source")), - Model: strings.TrimSpace(r.FormValue("model")), - Version: strings.TrimSpace(r.FormValue("version")), - File: strings.TrimSpace(r.FormValue("file")), - } - if req.Source == "" { + // One form posts one source or fifteen. A sharded model is one + // decision, not one per shard, and the mirror fetches each file + // independently anyway. + sources, files := r.Form["source"], r.Form["file"] + if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" { return "", errors.New("a source URL is required") } - got, err := m.Fetch(ctx, req) - if err != nil { - return "", err + model := strings.TrimSpace(r.FormValue("model")) + version := strings.TrimSpace(r.FormValue("version")) + + var started int + for i, src := range sources { + src = strings.TrimSpace(src) + if src == "" { + continue + } + req := mirror.FetchRequest{Source: src, Model: model, Version: version} + if i < len(files) { + req.File = strings.TrimSpace(files[i]) + } + if _, err := m.Fetch(ctx, req); err != nil { + // Reported with what already went, so a set that fails on + // its fourth shard does not read as though nothing started. + return "", fmt.Errorf("%d of %d started, then %s: %w", + started, len(sources), mirror.LastPathElement(src), err) + } + started++ } // Named as started, not as done. The mirror answered 202 and the // bytes are still moving; a message saying "added" would invite // somebody to point the catalog at a partial file. - return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil + return fmt.Sprintf("started downloading %s/%s (%s)", + model, version, plural(started, "file", "files")), nil })) mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m, @@ -278,6 +300,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) { return } d.Repo = &got + var weights []hf.File for _, f := range got.Files { if !f.IsWeights() { // Thirty files of which four are worth gigabytes. The rest are @@ -285,15 +308,22 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) { // somebody came for. continue } - d.RepoFiles = append(d.RepoFiles, repoFile{ - File: f, - Source: got.DownloadURL("", f.Path), - Model: defaultModelName(got.ID), + weights = append(weights, f) + } + for _, set := range hf.Group(weights) { + rs := repoSet{ + FileSet: set, + Model: defaultModelName(got.ID), // The revision, not the date: two fetches of the same commit // land in the same place, and a repository re-uploaded under // the same branch lands somewhere else. Version: got.ShortRevision(), - }) + } + for _, f := range set.Files { + rs.Sources = append(rs.Sources, + repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path}) + } + d.RepoSets = append(d.RepoSets, rs) } return } diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index 4b91cc94..1327973a 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -52,6 +52,7 @@ type fakeMirror struct { models []mirror.Status listErr error fetched mirror.FetchRequest + all []mirror.FetchRequest fetchErr error removed [2]string removeErr error @@ -68,6 +69,7 @@ func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) { func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) { f.fetched = r + f.all = append(f.all, r) if f.fetchErr != nil { return mirror.FetchResponse{}, f.fetchErr } @@ -435,3 +437,96 @@ func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) { t.Error("fetching by hand must still be there") } } + +func shardedRepo() *fakeRegistry { + return &fakeRegistry{repo: hf.Repo{ + ID: "Qwen/Qwen2.5-7B-Instruct", Revision: "abc123def456", + Files: []hf.File{ + {Path: "model-00001-of-00004.safetensors", Bytes: 4_000_000_000}, + {Path: "model-00002-of-00004.safetensors", Bytes: 4_000_000_000}, + {Path: "model-00003-of-00004.safetensors", Bytes: 4_000_000_000}, + {Path: "model-00004-of-00004.safetensors", Bytes: 3_000_000_000}, + {Path: "README.md", Bytes: 900}, + }, + }} +} + +func TestAShardedModelIsOneRowAndOneButton(t *testing.T) { + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, shardedRepo(), true), + "/admin/mirror?repo=Qwen/Qwen2.5-7B-Instruct").Body.String() + + // Fifteen shards is one decision, not fifteen. Four here, and one form. + if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 { + // One for the set, one for the manual form at the bottom. + t.Errorf("expected the four shards as a single form, found %d forms", n) + } + if !strings.Contains(body, "4 shards, fetched together") { + t.Errorf("the row should say it is a set; got %s", body) + } + // Summed, so the operator sees what the disk is in for. + if !strings.Contains(body, "14.0 GiB") { + t.Error("the set should total its shards") + } +} + +func TestFetchingASetStartsEveryShard(t *testing.T) { + m := &fakeMirror{} + form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"abc123de"}} + for i, p := range []string{"model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"} { + form.Add("source", "https://hf/resolve/abc123def456/"+p) + form.Add("file", p) + _ = i + } + body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true), + "/admin/mirror/fetch", form).Body.String() + + if len(m.all) != 2 { + t.Fatalf("expected both shards to be started, got %d", len(m.all)) + } + // Each shard keeps its own filename under one model and version, which is + // what makes them one version on disk. + if m.all[0].File == m.all[1].File { + t.Errorf("the shards collided on %q", m.all[0].File) + } + if m.all[0].Model != "qwen" || m.all[1].Version != "abc123de" { + t.Errorf("the set should share one model and version: %+v", m.all) + } + if !strings.Contains(body, "2 files") { + t.Errorf("the message should say how many started; got %s", body) + } +} + +func TestASetThatFailsPartWaySaysHowFarItGot(t *testing.T) { + m := &fakeMirror{fetchErr: errors.New("the mirror is full")} + form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"v"}} + form.Add("source", "https://hf/a.safetensors") + form.Add("file", "a.safetensors") + + body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true), + "/admin/mirror/fetch", form).Body.String() + + // "0 of 1 started, then …" rather than a bare error: a set that fails on + // its fourth shard has three arriving, and reading it as nothing started + // would send somebody to fetch them all again. + if !strings.Contains(body, "0 of 1 started") { + t.Errorf("the message should say how far it got; got %s", body) + } +} + +func TestAlternativeQuantisationsStaySeparateRows(t *testing.T) { + reg := &fakeRegistry{repo: hf.Repo{ + ID: "Qwen/X", Revision: "abc", + Files: []hf.File{ + {Path: "m-q4_k_m.gguf", Bytes: 1}, + {Path: "m-q8_0.gguf", Bytes: 2}, + }, + }} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), + "/admin/mirror?repo=Qwen/X").Body.String() + + // Two GGUFs are alternatives, not halves. Offering "fetch all" here would + // spend gigabytes on a copy nobody asked for. + if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 3 { + t.Errorf("expected a form each plus the manual one, found %d", n) + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index e435d843..c8276536 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -120,18 +120,20 @@ directory, or give the mirror a token.
{{if .Repo}}| File | Size | Published checksum | ||
|---|---|---|---|---|
{{.Path}} |
+||||
| What | Size | Published checksum | ||
{{.Name}}
+ {{if .Shards}}{{.Shards}} shards, fetched together{{end}} |
{{bytes .Bytes}} | -{{if .SHA256}}{{slice .SHA256 0 12}}{{else}}—{{end}} |
+ {{with .SHA256}}{{slice . 0 12}}{{else}}—{{end}} |
Only the weights are listed; a repository is mostly a README and
-a tokenizer. The source URL is pinned to {{.Repo.ShortRevision}}
-rather than to a branch, so what arrives is what is listed here even if the
-branch moves in between — and the checksum above is what the registry publishes,
-to compare with what the mirror computes once it has the file.
The source URLs are pinned to
+{{.Repo.ShortRevision}} rather than to a branch, so what arrives is
+what is listed here even if the branch moves in between — and a single file's
+checksum above is what the registry publishes, to compare with what the mirror
+computes once it has it.
No weight files in {{.Repo.ID}}.
{{end}}