The explorer listed every weight file with its own button. For a repository of alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded model it is fifteen clicks to fetch one thing, with nothing stopping you from losing count and leaving a version with fourteen of its parts. Files are grouped into sets now. A run of "-00002-of-00004" is one row and one button; everything else stays its own row. Not "fetch all the weights", which would be wrong half the time: two safetensors files are halves of one thing and useless apart, two GGUF files are usually alternatives and fetching both spends gigabytes on a copy nobody asked for. The set is the unit that is whole. A repository can hold both the split and the joined copy of the same quantisation — the real Qwen coder repos do — so a shard set keeps the -of- notation in its name rather than collapsing onto the name of the whole file beside it. A set publishes no single checksum. Several files have several, and showing one of them invites comparing it against the wrong thing. The form posts every source in the set, and a failure part way says how many already started: a set that fails on its fourth shard has three arriving, and reading that as "nothing happened" sends somebody to fetch them all again. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
190 lines
6.2 KiB
Go
190 lines
6.2 KiB
Go
package hf
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// registry stands in for Hugging Face, answering the two endpoints used.
|
|
func registry(t *testing.T, h http.HandlerFunc) *Client {
|
|
t.Helper()
|
|
srv := httptest.NewServer(h)
|
|
t.Cleanup(srv.Close)
|
|
return New(srv.URL, time.Second)
|
|
}
|
|
|
|
func TestTheRealSizeOfAWeightsFileIsTheLFSOne(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/tree/abc123") {
|
|
_, _ = w.Write([]byte(`[
|
|
{"type":"file","path":"README.md","size":900,"oid":"gitblob"},
|
|
{"type":"file","path":"model.gguf","size":133,
|
|
"lfs":{"oid":"deadbeef","size":1117320736}}]`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`))
|
|
})
|
|
|
|
repo, err := c.Repo(t.Context(), "Qwen/X")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var gguf File
|
|
for _, f := range repo.Files {
|
|
if f.Path == "model.gguf" {
|
|
gguf = f
|
|
}
|
|
}
|
|
// The plain size of an LFS file is the size of the pointer. Reading that
|
|
// one reports a gigabyte model as 133 bytes, which is the number somebody
|
|
// would then plan disk against.
|
|
if gguf.Bytes != 1117320736 {
|
|
t.Errorf("expected the LFS size, got %d", gguf.Bytes)
|
|
}
|
|
if gguf.SHA256 != "deadbeef" {
|
|
t.Errorf("expected the published checksum, got %q", gguf.SHA256)
|
|
}
|
|
}
|
|
|
|
func TestASmallFileHasNoChecksum(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.Contains(r.URL.Path, "/tree/") {
|
|
_, _ = w.Write([]byte(`[{"type":"file","path":"README.md","size":900,"oid":"gitblob"}]`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`))
|
|
})
|
|
|
|
repo, _ := c.Repo(t.Context(), "Qwen/X")
|
|
// A git blob hash is not a checksum of the content, so reporting it as one
|
|
// would invite somebody to compare it with what the mirror computes and
|
|
// conclude the download was corrupt.
|
|
if repo.Files[0].SHA256 != "" {
|
|
t.Errorf("a git-stored file should publish no checksum, got %q", repo.Files[0].SHA256)
|
|
}
|
|
}
|
|
|
|
func TestTheDownloadURLIsPinnedToTheRevisionThatWasListed(t *testing.T) {
|
|
r := Repo{ID: "Qwen/X", Revision: "abc123def456"}
|
|
got := r.DownloadURL("https://hf.example", "model.gguf")
|
|
|
|
// "main" would mean the mirror fetches whatever is there when it gets round
|
|
// to it, which is not necessarily what was on screen when the button was
|
|
// pressed.
|
|
if !strings.Contains(got, "/resolve/abc123def456/") {
|
|
t.Errorf("the URL should pin the revision; got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAGatedRepositorySaysSoRatherThanBlamingTheNetwork(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
})
|
|
|
|
_, err := c.Repo(t.Context(), "meta/gated")
|
|
if err == nil || !strings.Contains(err.Error(), "gated or private") {
|
|
t.Errorf("expected the reason to be named; got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOneGatedResultDoesNotHideThePage(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
// The field is documented as a boolean and is a word in practice.
|
|
// Decoding it strictly fails the whole response, so one gated model in
|
|
// a page of twenty would hide the other nineteen.
|
|
_, _ = w.Write([]byte(`[{"id":"a","gated":false},{"id":"b","gated":"manual"}]`))
|
|
})
|
|
|
|
got, err := c.Search(t.Context(), "qwen", 20)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 2 {
|
|
t.Fatalf("expected both results, got %d", len(got))
|
|
}
|
|
if got[0].Gated.Is() || !got[1].Gated.Is() {
|
|
t.Errorf("gating did not decode: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestSomethingThatIsNotARepositoryIsRefusedBeforeTheCall(t *testing.T) {
|
|
called := false
|
|
c := registry(t, func(http.ResponseWriter, *http.Request) { called = true })
|
|
|
|
if _, err := c.Repo(t.Context(), "qwen"); err == nil {
|
|
t.Error("a bare name is not owner/name and should be refused")
|
|
}
|
|
if called {
|
|
t.Error("nothing should have been requested")
|
|
}
|
|
}
|
|
|
|
func TestWeightsAreToldApartFromDocumentation(t *testing.T) {
|
|
// Crude and right: the registry does not label them, and an operator
|
|
// filling a mirror wants the four files out of thirty worth gigabytes.
|
|
for _, f := range []string{"model.gguf", "model-00001-of-00002.safetensors", "pytorch_model.bin"} {
|
|
if !(File{Path: f}).IsWeights() {
|
|
t.Errorf("%s should count as weights", f)
|
|
}
|
|
}
|
|
for _, f := range []string{"README.md", "config.json", ".gitattributes", "tokenizer.model"} {
|
|
if (File{Path: f}).IsWeights() {
|
|
t.Errorf("%s should not count as weights", f)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestShardsBecomeOneSetAndAlternativesStaySeparate(t *testing.T) {
|
|
// Straight from a real repository, which holds both the split and the
|
|
// joined copy of the same quantisation.
|
|
sets := Group([]File{
|
|
{Path: "qwen-fp16-00001-of-00004.gguf", Bytes: 1},
|
|
{Path: "qwen-fp16-00002-of-00004.gguf", Bytes: 1},
|
|
{Path: "qwen-fp16-00003-of-00004.gguf", Bytes: 1},
|
|
{Path: "qwen-fp16-00004-of-00004.gguf", Bytes: 1},
|
|
{Path: "qwen-fp16.gguf", Bytes: 4},
|
|
{Path: "qwen-q4_k_m.gguf", Bytes: 2},
|
|
})
|
|
|
|
if len(sets) != 3 {
|
|
t.Fatalf("expected the shards as one and the two whole files apart, got %d: %+v",
|
|
len(sets), sets)
|
|
}
|
|
if sets[0].Shards != 4 || sets[0].Bytes != 4 {
|
|
t.Errorf("the shard set should total its parts, got %+v", sets[0])
|
|
}
|
|
// Two GGUFs in a repository are alternatives: fetching both is wasting
|
|
// gigabytes on a copy nobody asked for.
|
|
if sets[1].Shards != 0 || sets[2].Shards != 0 {
|
|
t.Errorf("whole files are not shard sets: %+v %+v", sets[1], sets[2])
|
|
}
|
|
}
|
|
|
|
func TestASplitAndAJoinedCopyDoNotReadAsOneEntry(t *testing.T) {
|
|
sets := Group([]File{
|
|
{Path: "m-q4-00001-of-00002.gguf"}, {Path: "m-q4-00002-of-00002.gguf"},
|
|
{Path: "m-q4.gguf"},
|
|
})
|
|
if sets[0].Name == sets[1].Name {
|
|
t.Errorf("the split and the joined copy collided on %q", sets[0].Name)
|
|
}
|
|
if !strings.Contains(sets[0].Name, "-of-00002") {
|
|
t.Errorf("a shard set should keep the notation; got %q", sets[0].Name)
|
|
}
|
|
}
|
|
|
|
func TestAShardSetPublishesNoSingleChecksum(t *testing.T) {
|
|
sets := Group([]File{
|
|
{Path: "m-00001-of-00002.safetensors", SHA256: "aaa"},
|
|
{Path: "m-00002-of-00002.safetensors", SHA256: "bbb"},
|
|
})
|
|
// Several files have several checksums and no single value for the set;
|
|
// showing one invites comparing it against the wrong thing.
|
|
if got := sets[0].SHA256(); got != "" {
|
|
t.Errorf("expected no checksum for a set, got %q", got)
|
|
}
|
|
}
|