CI / test (push) Successful in 1m41s
Mirroring anything meant knowing a repository existed, knowing which of its thirty files were the weights, and assembling a resolve URL by hand. The mirror page had a text box for that URL and no way to find one. It now searches Hugging Face and lists a repository's weight files with their sizes and published checksums, each with a button that fills the fetch form. Only weight files. A repository is mostly a README and a tokenizer, and listing those buries the four entries somebody came for. The size shown is the LFS size, not the plain one. For an LFS-stored file — every weights file — the plain field is the size of the pointer, so the other one reports a gigabyte model as 133 bytes, and that is the number somebody would then plan disk against. The source URL is pinned to the commit the listing came from rather than to a branch, so what the mirror fetches is what was on screen. A repository re-uploaded under the same branch otherwise leaves two different sets of weights with identical provenance and no way to tell them apart. The version defaults to that revision for the same reason. The registry's own failures are reported on their own line. It is somebody else's server on the other side of the internet, and it being down says nothing about the mirror — which is what the page is for and has to keep working regardless. With no registry configured there is no search box at all, rather than one that can only time out. Gating decodes from a bool or a string, because the field is documented as the first and is the second in practice: strict decoding fails the whole response, so one gated model would hide the other nineteen in a page of results. And a bug this made obvious: downloads were keyed by model and version, so the second shard of a sharded model was refused as "already fetching" until the first had finished. A fifteen-shard model had to be pulled one file at a time, each waiting on the last. Keyed by file now — which is what the single-flight was ever protecting, since the race is two writers on one .part. A version made of several files reports as the worst of them: it is not present until every shard is, and one shard failing makes the version unusable however many others arrived. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
139 lines
4.4 KiB
Go
139 lines
4.4 KiB
Go
package hf
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// registry stands in for Hugging Face, answering the two endpoints used.
|
|
func registry(t *testing.T, h http.HandlerFunc) *Client {
|
|
t.Helper()
|
|
srv := httptest.NewServer(h)
|
|
t.Cleanup(srv.Close)
|
|
return New(srv.URL, time.Second)
|
|
}
|
|
|
|
func TestTheRealSizeOfAWeightsFileIsTheLFSOne(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.HasSuffix(r.URL.Path, "/tree/abc123") {
|
|
_, _ = w.Write([]byte(`[
|
|
{"type":"file","path":"README.md","size":900,"oid":"gitblob"},
|
|
{"type":"file","path":"model.gguf","size":133,
|
|
"lfs":{"oid":"deadbeef","size":1117320736}}]`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`))
|
|
})
|
|
|
|
repo, err := c.Repo(t.Context(), "Qwen/X")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var gguf File
|
|
for _, f := range repo.Files {
|
|
if f.Path == "model.gguf" {
|
|
gguf = f
|
|
}
|
|
}
|
|
// The plain size of an LFS file is the size of the pointer. Reading that
|
|
// one reports a gigabyte model as 133 bytes, which is the number somebody
|
|
// would then plan disk against.
|
|
if gguf.Bytes != 1117320736 {
|
|
t.Errorf("expected the LFS size, got %d", gguf.Bytes)
|
|
}
|
|
if gguf.SHA256 != "deadbeef" {
|
|
t.Errorf("expected the published checksum, got %q", gguf.SHA256)
|
|
}
|
|
}
|
|
|
|
func TestASmallFileHasNoChecksum(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if strings.Contains(r.URL.Path, "/tree/") {
|
|
_, _ = w.Write([]byte(`[{"type":"file","path":"README.md","size":900,"oid":"gitblob"}]`))
|
|
return
|
|
}
|
|
_, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`))
|
|
})
|
|
|
|
repo, _ := c.Repo(t.Context(), "Qwen/X")
|
|
// A git blob hash is not a checksum of the content, so reporting it as one
|
|
// would invite somebody to compare it with what the mirror computes and
|
|
// conclude the download was corrupt.
|
|
if repo.Files[0].SHA256 != "" {
|
|
t.Errorf("a git-stored file should publish no checksum, got %q", repo.Files[0].SHA256)
|
|
}
|
|
}
|
|
|
|
func TestTheDownloadURLIsPinnedToTheRevisionThatWasListed(t *testing.T) {
|
|
r := Repo{ID: "Qwen/X", Revision: "abc123def456"}
|
|
got := r.DownloadURL("https://hf.example", "model.gguf")
|
|
|
|
// "main" would mean the mirror fetches whatever is there when it gets round
|
|
// to it, which is not necessarily what was on screen when the button was
|
|
// pressed.
|
|
if !strings.Contains(got, "/resolve/abc123def456/") {
|
|
t.Errorf("the URL should pin the revision; got %s", got)
|
|
}
|
|
}
|
|
|
|
func TestAGatedRepositorySaysSoRatherThanBlamingTheNetwork(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
})
|
|
|
|
_, err := c.Repo(t.Context(), "meta/gated")
|
|
if err == nil || !strings.Contains(err.Error(), "gated or private") {
|
|
t.Errorf("expected the reason to be named; got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOneGatedResultDoesNotHideThePage(t *testing.T) {
|
|
c := registry(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
// The field is documented as a boolean and is a word in practice.
|
|
// Decoding it strictly fails the whole response, so one gated model in
|
|
// a page of twenty would hide the other nineteen.
|
|
_, _ = w.Write([]byte(`[{"id":"a","gated":false},{"id":"b","gated":"manual"}]`))
|
|
})
|
|
|
|
got, err := c.Search(t.Context(), "qwen", 20)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != 2 {
|
|
t.Fatalf("expected both results, got %d", len(got))
|
|
}
|
|
if got[0].Gated.Is() || !got[1].Gated.Is() {
|
|
t.Errorf("gating did not decode: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestSomethingThatIsNotARepositoryIsRefusedBeforeTheCall(t *testing.T) {
|
|
called := false
|
|
c := registry(t, func(http.ResponseWriter, *http.Request) { called = true })
|
|
|
|
if _, err := c.Repo(t.Context(), "qwen"); err == nil {
|
|
t.Error("a bare name is not owner/name and should be refused")
|
|
}
|
|
if called {
|
|
t.Error("nothing should have been requested")
|
|
}
|
|
}
|
|
|
|
func TestWeightsAreToldApartFromDocumentation(t *testing.T) {
|
|
// Crude and right: the registry does not label them, and an operator
|
|
// filling a mirror wants the four files out of thirty worth gigabytes.
|
|
for _, f := range []string{"model.gguf", "model-00001-of-00002.safetensors", "pytorch_model.bin"} {
|
|
if !(File{Path: f}).IsWeights() {
|
|
t.Errorf("%s should count as weights", f)
|
|
}
|
|
}
|
|
for _, f := range []string{"README.md", "config.json", ".gitattributes", "tokenizer.model"} {
|
|
if (File{Path: f}).IsWeights() {
|
|
t.Errorf("%s should not count as weights", f)
|
|
}
|
|
}
|
|
}
|