The explorer listed every weight file with its own button. For a repository of alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded model it is fifteen clicks to fetch one thing, with nothing stopping you from losing count and leaving a version with fourteen of its parts. Files are grouped into sets now. A run of "-00002-of-00004" is one row and one button; everything else stays its own row. Not "fetch all the weights", which would be wrong half the time: two safetensors files are halves of one thing and useless apart, two GGUF files are usually alternatives and fetching both spends gigabytes on a copy nobody asked for. The set is the unit that is whole. A repository can hold both the split and the joined copy of the same quantisation — the real Qwen coder repos do — so a shard set keeps the -of- notation in its name rather than collapsing onto the name of the whole file beside it. A set publishes no single checksum. Several files have several, and showing one of them invites comparing it against the wrong thing. The form posts every source in the set, and a failure part way says how many already started: a set that fails on its fourth shard has three arriving, and reading that as "nothing happened" sends somebody to fetch them all again. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ed530133b6
commit
a588646b7c
5 files changed
+277
-31
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -257,3 +258,66 @@ func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shardPattern matches the "-00002-of-00004" a split file carries before its
|
||||
// extension. Five digits in practice, but not required: the count is what
|
||||
// matters and a repository that used three would otherwise list every part as
|
||||
// a separate model.
|
||||
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
|
||||
|
||||
// FileSet is one thing worth fetching: a file, or every shard of one.
|
||||
//
|
||||
// The distinction the page turns on. Two GGUF files in a repository are usually
|
||||
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
|
||||
// halves of the same weights and useless apart. Fetching "all the weights" is
|
||||
// right for the second and wasteful for the first, so neither is offered:
|
||||
// a set is offered, and a set is whole.
|
||||
type FileSet struct {
|
||||
// Name is what to call it. For shards it keeps the -of- notation, because
|
||||
// a repository can hold both the split and the joined copy of the same
|
||||
// quantisation and they must not read as one entry.
|
||||
Name string
|
||||
Files []File
|
||||
Bytes int64
|
||||
Shards int
|
||||
}
|
||||
|
||||
// SHA256 is the checksum when there is one to give.
|
||||
//
|
||||
// Empty for a shard set: several files have several checksums and no single
|
||||
// value for the set, and showing one of them invites comparing it against the
|
||||
// wrong thing.
|
||||
func (s FileSet) SHA256() string {
|
||||
if len(s.Files) == 1 {
|
||||
return s.Files[0].SHA256
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Group collects shards into sets, leaving every other file on its own.
|
||||
//
|
||||
// Order is preserved from the listing, so a repository that sorts its files
|
||||
// sensibly still reads sensibly here.
|
||||
func Group(files []File) []FileSet {
|
||||
var out []FileSet
|
||||
at := map[string]int{}
|
||||
for _, f := range files {
|
||||
m := shardPattern.FindStringSubmatch(f.Path)
|
||||
if m == nil {
|
||||
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
|
||||
continue
|
||||
}
|
||||
prefix, total, ext := m[1], m[3], m[4]
|
||||
name := prefix + "-*-of-" + total + ext
|
||||
i, ok := at[name]
|
||||
if !ok {
|
||||
at[name] = len(out)
|
||||
out = append(out, FileSet{Name: name})
|
||||
i = at[name]
|
||||
}
|
||||
out[i].Files = append(out[i].Files, f)
|
||||
out[i].Bytes += f.Bytes
|
||||
out[i].Shards = len(out[i].Files)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -136,3 +136,54 @@ func TestWeightsAreToldApartFromDocumentation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardsBecomeOneSetAndAlternativesStaySeparate(t *testing.T) {
|
||||
// Straight from a real repository, which holds both the split and the
|
||||
// joined copy of the same quantisation.
|
||||
sets := Group([]File{
|
||||
{Path: "qwen-fp16-00001-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00002-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00003-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00004-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16.gguf", Bytes: 4},
|
||||
{Path: "qwen-q4_k_m.gguf", Bytes: 2},
|
||||
})
|
||||
|
||||
if len(sets) != 3 {
|
||||
t.Fatalf("expected the shards as one and the two whole files apart, got %d: %+v",
|
||||
len(sets), sets)
|
||||
}
|
||||
if sets[0].Shards != 4 || sets[0].Bytes != 4 {
|
||||
t.Errorf("the shard set should total its parts, got %+v", sets[0])
|
||||
}
|
||||
// Two GGUFs in a repository are alternatives: fetching both is wasting
|
||||
// gigabytes on a copy nobody asked for.
|
||||
if sets[1].Shards != 0 || sets[2].Shards != 0 {
|
||||
t.Errorf("whole files are not shard sets: %+v %+v", sets[1], sets[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestASplitAndAJoinedCopyDoNotReadAsOneEntry(t *testing.T) {
|
||||
sets := Group([]File{
|
||||
{Path: "m-q4-00001-of-00002.gguf"}, {Path: "m-q4-00002-of-00002.gguf"},
|
||||
{Path: "m-q4.gguf"},
|
||||
})
|
||||
if sets[0].Name == sets[1].Name {
|
||||
t.Errorf("the split and the joined copy collided on %q", sets[0].Name)
|
||||
}
|
||||
if !strings.Contains(sets[0].Name, "-of-00002") {
|
||||
t.Errorf("a shard set should keep the notation; got %q", sets[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShardSetPublishesNoSingleChecksum(t *testing.T) {
|
||||
sets := Group([]File{
|
||||
{Path: "m-00001-of-00002.safetensors", SHA256: "aaa"},
|
||||
{Path: "m-00002-of-00002.safetensors", SHA256: "bbb"},
|
||||
})
|
||||
// Several files have several checksums and no single value for the set;
|
||||
// showing one invites comparing it against the wrong thing.
|
||||
if got := sets[0].SHA256(); got != "" {
|
||||
t.Errorf("expected no checksum for a set, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -36,10 +37,13 @@ type Registry interface {
|
||||
Repo(ctx context.Context, id string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// repoFile is one file of a browsed repository, ready to be mirrored.
|
||||
type repoFile struct {
|
||||
hf.File
|
||||
Source string
|
||||
// repoSet is one thing worth fetching: a file, or every shard of one.
|
||||
type repoSet struct {
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
// decisions, it is one.
|
||||
Sources []repoSource
|
||||
// Model and Version are what the fetch form starts with. Editable, because
|
||||
// the name a catalog entry uses is the operator's to choose and the
|
||||
// registry's idea of it is only a good guess.
|
||||
@@ -47,6 +51,8 @@ type repoFile struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
type repoSource struct{ URL, Path string }
|
||||
|
||||
// mirrorRow is one model version as the page shows it.
|
||||
type mirrorRow struct {
|
||||
mirror.Status
|
||||
@@ -79,7 +85,7 @@ type adminMirrorData struct {
|
||||
Query string
|
||||
Results []hf.Model
|
||||
Repo *hf.Repo
|
||||
RepoFiles []repoFile
|
||||
RepoSets []repoSet
|
||||
RegistryErr string
|
||||
|
||||
Message string
|
||||
@@ -112,23 +118,39 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
req := mirror.FetchRequest{
|
||||
Source: strings.TrimSpace(r.FormValue("source")),
|
||||
Model: strings.TrimSpace(r.FormValue("model")),
|
||||
Version: strings.TrimSpace(r.FormValue("version")),
|
||||
File: strings.TrimSpace(r.FormValue("file")),
|
||||
}
|
||||
if req.Source == "" {
|
||||
// One form posts one source or fifteen. A sharded model is one
|
||||
// decision, not one per shard, and the mirror fetches each file
|
||||
// independently anyway.
|
||||
sources, files := r.Form["source"], r.Form["file"]
|
||||
if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" {
|
||||
return "", errors.New("a source URL is required")
|
||||
}
|
||||
got, err := m.Fetch(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
model := strings.TrimSpace(r.FormValue("model"))
|
||||
version := strings.TrimSpace(r.FormValue("version"))
|
||||
|
||||
var started int
|
||||
for i, src := range sources {
|
||||
src = strings.TrimSpace(src)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
req := mirror.FetchRequest{Source: src, Model: model, Version: version}
|
||||
if i < len(files) {
|
||||
req.File = strings.TrimSpace(files[i])
|
||||
}
|
||||
if _, err := m.Fetch(ctx, req); err != nil {
|
||||
// Reported with what already went, so a set that fails on
|
||||
// its fourth shard does not read as though nothing started.
|
||||
return "", fmt.Errorf("%d of %d started, then %s: %w",
|
||||
started, len(sources), mirror.LastPathElement(src), err)
|
||||
}
|
||||
started++
|
||||
}
|
||||
// Named as started, not as done. The mirror answered 202 and the
|
||||
// bytes are still moving; a message saying "added" would invite
|
||||
// somebody to point the catalog at a partial file.
|
||||
return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil
|
||||
return fmt.Sprintf("started downloading %s/%s (%s)",
|
||||
model, version, plural(started, "file", "files")), nil
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
|
||||
@@ -278,6 +300,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
return
|
||||
}
|
||||
d.Repo = &got
|
||||
var weights []hf.File
|
||||
for _, f := range got.Files {
|
||||
if !f.IsWeights() {
|
||||
// Thirty files of which four are worth gigabytes. The rest are
|
||||
@@ -285,15 +308,22 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
// somebody came for.
|
||||
continue
|
||||
}
|
||||
d.RepoFiles = append(d.RepoFiles, repoFile{
|
||||
File: f,
|
||||
Source: got.DownloadURL("", f.Path),
|
||||
Model: defaultModelName(got.ID),
|
||||
weights = append(weights, f)
|
||||
}
|
||||
for _, set := range hf.Group(weights) {
|
||||
rs := repoSet{
|
||||
FileSet: set,
|
||||
Model: defaultModelName(got.ID),
|
||||
// The revision, not the date: two fetches of the same commit
|
||||
// land in the same place, and a repository re-uploaded under
|
||||
// the same branch lands somewhere else.
|
||||
Version: got.ShortRevision(),
|
||||
})
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
}
|
||||
d.RepoSets = append(d.RepoSets, rs)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ type fakeMirror struct {
|
||||
models []mirror.Status
|
||||
listErr error
|
||||
fetched mirror.FetchRequest
|
||||
all []mirror.FetchRequest
|
||||
fetchErr error
|
||||
removed [2]string
|
||||
removeErr error
|
||||
@@ -68,6 +69,7 @@ func (f *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
|
||||
|
||||
func (f *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
|
||||
f.fetched = r
|
||||
f.all = append(f.all, r)
|
||||
if f.fetchErr != nil {
|
||||
return mirror.FetchResponse{}, f.fetchErr
|
||||
}
|
||||
@@ -435,3 +437,96 @@ func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) {
|
||||
t.Error("fetching by hand must still be there")
|
||||
}
|
||||
}
|
||||
|
||||
func shardedRepo() *fakeRegistry {
|
||||
return &fakeRegistry{repo: hf.Repo{
|
||||
ID: "Qwen/Qwen2.5-7B-Instruct", Revision: "abc123def456",
|
||||
Files: []hf.File{
|
||||
{Path: "model-00001-of-00004.safetensors", Bytes: 4_000_000_000},
|
||||
{Path: "model-00002-of-00004.safetensors", Bytes: 4_000_000_000},
|
||||
{Path: "model-00003-of-00004.safetensors", Bytes: 4_000_000_000},
|
||||
{Path: "model-00004-of-00004.safetensors", Bytes: 3_000_000_000},
|
||||
{Path: "README.md", Bytes: 900},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAShardedModelIsOneRowAndOneButton(t *testing.T) {
|
||||
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, shardedRepo(), true),
|
||||
"/admin/mirror?repo=Qwen/Qwen2.5-7B-Instruct").Body.String()
|
||||
|
||||
// Fifteen shards is one decision, not fifteen. Four here, and one form.
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 {
|
||||
// One for the set, one for the manual form at the bottom.
|
||||
t.Errorf("expected the four shards as a single form, found %d forms", n)
|
||||
}
|
||||
if !strings.Contains(body, "4 shards, fetched together") {
|
||||
t.Errorf("the row should say it is a set; got %s", body)
|
||||
}
|
||||
// Summed, so the operator sees what the disk is in for.
|
||||
if !strings.Contains(body, "14.0 GiB") {
|
||||
t.Error("the set should total its shards")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchingASetStartsEveryShard(t *testing.T) {
|
||||
m := &fakeMirror{}
|
||||
form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"abc123de"}}
|
||||
for i, p := range []string{"model-00001-of-00004.safetensors", "model-00002-of-00004.safetensors"} {
|
||||
form.Add("source", "https://hf/resolve/abc123def456/"+p)
|
||||
form.Add("file", p)
|
||||
_ = i
|
||||
}
|
||||
body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
|
||||
"/admin/mirror/fetch", form).Body.String()
|
||||
|
||||
if len(m.all) != 2 {
|
||||
t.Fatalf("expected both shards to be started, got %d", len(m.all))
|
||||
}
|
||||
// Each shard keeps its own filename under one model and version, which is
|
||||
// what makes them one version on disk.
|
||||
if m.all[0].File == m.all[1].File {
|
||||
t.Errorf("the shards collided on %q", m.all[0].File)
|
||||
}
|
||||
if m.all[0].Model != "qwen" || m.all[1].Version != "abc123de" {
|
||||
t.Errorf("the set should share one model and version: %+v", m.all)
|
||||
}
|
||||
if !strings.Contains(body, "2 files") {
|
||||
t.Errorf("the message should say how many started; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASetThatFailsPartWaySaysHowFarItGot(t *testing.T) {
|
||||
m := &fakeMirror{fetchErr: errors.New("the mirror is full")}
|
||||
form := url.Values{"csrf": {"tok"}, "model": {"qwen"}, "version": {"v"}}
|
||||
form.Add("source", "https://hf/a.safetensors")
|
||||
form.Add("file", "a.safetensors")
|
||||
|
||||
body := mirrorPost(t, mirrorPageWith(t, m, shardedRepo(), true),
|
||||
"/admin/mirror/fetch", form).Body.String()
|
||||
|
||||
// "0 of 1 started, then …" rather than a bare error: a set that fails on
|
||||
// its fourth shard has three arriving, and reading it as nothing started
|
||||
// would send somebody to fetch them all again.
|
||||
if !strings.Contains(body, "0 of 1 started") {
|
||||
t.Errorf("the message should say how far it got; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlternativeQuantisationsStaySeparateRows(t *testing.T) {
|
||||
reg := &fakeRegistry{repo: hf.Repo{
|
||||
ID: "Qwen/X", Revision: "abc",
|
||||
Files: []hf.File{
|
||||
{Path: "m-q4_k_m.gguf", Bytes: 1},
|
||||
{Path: "m-q8_0.gguf", Bytes: 2},
|
||||
},
|
||||
}}
|
||||
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
|
||||
"/admin/mirror?repo=Qwen/X").Body.String()
|
||||
|
||||
// Two GGUFs are alternatives, not halves. Offering "fetch all" here would
|
||||
// spend gigabytes on a copy nobody asked for.
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 3 {
|
||||
t.Errorf("expected a form each plus the manual one, found %d", n)
|
||||
}
|
||||
}
|
||||
@@ -120,18 +120,20 @@ directory, or give the mirror a token.</p>
|
||||
|
||||
{{if .Repo}}
|
||||
<h3>{{.Repo.ID}} <span class="opt">at {{.Repo.ShortRevision}}</span></h3>
|
||||
{{if .RepoFiles}}
|
||||
{{if .RepoSets}}
|
||||
<table>
|
||||
<tr><th>File</th><th>Size</th><th>Published checksum</th><th></th></tr>
|
||||
{{range .RepoFiles}}<tr>
|
||||
<td><code>{{.Path}}</code></td>
|
||||
<tr><th>What</th><th>Size</th><th>Published checksum</th><th></th></tr>
|
||||
{{range .RepoSets}}<tr>
|
||||
<td><code>{{.Name}}</code>
|
||||
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}</td>
|
||||
<td class="num">{{bytes .Bytes}}</td>
|
||||
<td>{{if .SHA256}}<code title="{{.SHA256}}">{{slice .SHA256 0 12}}</code>{{else}}—{{end}}</td>
|
||||
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
|
||||
<td>
|
||||
<form method="post" action="/admin/mirror/fetch" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="source" value="{{.Source}}">
|
||||
{{range .Sources}}<input type="hidden" name="source" value="{{.URL}}">
|
||||
<input type="hidden" name="file" value="{{.Path}}">
|
||||
{{end}}
|
||||
<input type="text" name="model" value="{{.Model}}" size="24">
|
||||
<input type="text" name="version" value="{{.Version}}" size="10">
|
||||
<button type="submit">Fetch</button>
|
||||
@@ -140,10 +142,14 @@ directory, or give the mirror a token.</p>
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<p class="note">Only the weights are listed; a repository is mostly a README and
|
||||
a tokenizer. The source URL is pinned to <code>{{.Repo.ShortRevision}}</code>
|
||||
rather than to a branch, so what arrives is what is listed here even if the
|
||||
branch moves in between — and the checksum above is what the registry publishes,
|
||||
to compare with what the mirror computes once it has the file.</p>
|
||||
a tokenizer. Shards of one file are one row and one button — a fifteen-shard
|
||||
model is one decision, not fifteen. Two separate files are two rows because they
|
||||
are usually alternatives: one quantisation or the other, not both.</p>
|
||||
<p class="note">The source URLs are pinned to
|
||||
<code>{{.Repo.ShortRevision}}</code> rather than to a branch, so what arrives is
|
||||
what is listed here even if the branch moves in between — and a single file's
|
||||
checksum above is what the registry publishes, to compare with what the mirror
|
||||
computes once it has it.</p>
|
||||
{{else}}
|
||||
<p class="empty">No weight files in {{.Repo.ID}}.</p>
|
||||
{{end}}
|
||||
|
||||
Reference in new issue
Block a user