The explorer listed every weight file with its own button. For a repository of alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded model it is fifteen clicks to fetch one thing, with nothing stopping you from losing count and leaving a version with fourteen of its parts. Files are grouped into sets now. A run of "-00002-of-00004" is one row and one button; everything else stays its own row. Not "fetch all the weights", which would be wrong half the time: two safetensors files are halves of one thing and useless apart, two GGUF files are usually alternatives and fetching both spends gigabytes on a copy nobody asked for. The set is the unit that is whole. A repository can hold both the split and the joined copy of the same quantisation — the real Qwen coder repos do — so a shard set keeps the -of- notation in its name rather than collapsing onto the name of the whole file beside it. A set publishes no single checksum. Several files have several, and showing one of them invites comparing it against the wrong thing. The form posts every source in the set, and a failure part way says how many already started: a set that fails on its fourth shard has three arriving, and reading that as "nothing happened" sends somebody to fetch them all again. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ed530133b6
commit
a588646b7c
5 files changed
+277
-31
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -257,3 +258,66 @@ func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shardPattern matches the "-00002-of-00004" a split file carries before its
|
||||
// extension. Five digits in practice, but not required: the count is what
|
||||
// matters and a repository that used three would otherwise list every part as
|
||||
// a separate model.
|
||||
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
|
||||
|
||||
// FileSet is one thing worth fetching: a file, or every shard of one.
|
||||
//
|
||||
// The distinction the page turns on. Two GGUF files in a repository are usually
|
||||
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
|
||||
// halves of the same weights and useless apart. Fetching "all the weights" is
|
||||
// right for the second and wasteful for the first, so neither is offered:
|
||||
// a set is offered, and a set is whole.
|
||||
type FileSet struct {
|
||||
// Name is what to call it. For shards it keeps the -of- notation, because
|
||||
// a repository can hold both the split and the joined copy of the same
|
||||
// quantisation and they must not read as one entry.
|
||||
Name string
|
||||
Files []File
|
||||
Bytes int64
|
||||
Shards int
|
||||
}
|
||||
|
||||
// SHA256 is the checksum when there is one to give.
|
||||
//
|
||||
// Empty for a shard set: several files have several checksums and no single
|
||||
// value for the set, and showing one of them invites comparing it against the
|
||||
// wrong thing.
|
||||
func (s FileSet) SHA256() string {
|
||||
if len(s.Files) == 1 {
|
||||
return s.Files[0].SHA256
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Group collects shards into sets, leaving every other file on its own.
|
||||
//
|
||||
// Order is preserved from the listing, so a repository that sorts its files
|
||||
// sensibly still reads sensibly here.
|
||||
func Group(files []File) []FileSet {
|
||||
var out []FileSet
|
||||
at := map[string]int{}
|
||||
for _, f := range files {
|
||||
m := shardPattern.FindStringSubmatch(f.Path)
|
||||
if m == nil {
|
||||
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
|
||||
continue
|
||||
}
|
||||
prefix, total, ext := m[1], m[3], m[4]
|
||||
name := prefix + "-*-of-" + total + ext
|
||||
i, ok := at[name]
|
||||
if !ok {
|
||||
at[name] = len(out)
|
||||
out = append(out, FileSet{Name: name})
|
||||
i = at[name]
|
||||
}
|
||||
out[i].Files = append(out[i].Files, f)
|
||||
out[i].Bytes += f.Bytes
|
||||
out[i].Shards = len(out[i].Files)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -136,3 +136,54 @@ func TestWeightsAreToldApartFromDocumentation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardsBecomeOneSetAndAlternativesStaySeparate(t *testing.T) {
|
||||
// Straight from a real repository, which holds both the split and the
|
||||
// joined copy of the same quantisation.
|
||||
sets := Group([]File{
|
||||
{Path: "qwen-fp16-00001-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00002-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00003-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16-00004-of-00004.gguf", Bytes: 1},
|
||||
{Path: "qwen-fp16.gguf", Bytes: 4},
|
||||
{Path: "qwen-q4_k_m.gguf", Bytes: 2},
|
||||
})
|
||||
|
||||
if len(sets) != 3 {
|
||||
t.Fatalf("expected the shards as one and the two whole files apart, got %d: %+v",
|
||||
len(sets), sets)
|
||||
}
|
||||
if sets[0].Shards != 4 || sets[0].Bytes != 4 {
|
||||
t.Errorf("the shard set should total its parts, got %+v", sets[0])
|
||||
}
|
||||
// Two GGUFs in a repository are alternatives: fetching both is wasting
|
||||
// gigabytes on a copy nobody asked for.
|
||||
if sets[1].Shards != 0 || sets[2].Shards != 0 {
|
||||
t.Errorf("whole files are not shard sets: %+v %+v", sets[1], sets[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestASplitAndAJoinedCopyDoNotReadAsOneEntry(t *testing.T) {
|
||||
sets := Group([]File{
|
||||
{Path: "m-q4-00001-of-00002.gguf"}, {Path: "m-q4-00002-of-00002.gguf"},
|
||||
{Path: "m-q4.gguf"},
|
||||
})
|
||||
if sets[0].Name == sets[1].Name {
|
||||
t.Errorf("the split and the joined copy collided on %q", sets[0].Name)
|
||||
}
|
||||
if !strings.Contains(sets[0].Name, "-of-00002") {
|
||||
t.Errorf("a shard set should keep the notation; got %q", sets[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShardSetPublishesNoSingleChecksum(t *testing.T) {
|
||||
sets := Group([]File{
|
||||
{Path: "m-00001-of-00002.safetensors", SHA256: "aaa"},
|
||||
{Path: "m-00002-of-00002.safetensors", SHA256: "bbb"},
|
||||
})
|
||||
// Several files have several checksums and no single value for the set;
|
||||
// showing one invites comparing it against the wrong thing.
|
||||
if got := sets[0].SHA256(); got != "" {
|
||||
t.Errorf("expected no checksum for a set, got %q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user