// Package hf reads the Hugging Face model registry. // // Enough of it to answer two questions an operator has before mirroring a // model: what is there, and which file do I want. Nothing here downloads // anything — the mirror does that, over its own network, and the point of the // split is that the control plane never carries a model's bytes. // // Read-only and anonymous. A gated or private repository answers 401 to this // client, which is reported rather than worked around: a token belongs to the // machine that fetches the weights, not to the one browsing a catalogue. package hf import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" ) // DefaultEndpoint is the public registry. const DefaultEndpoint = "https://huggingface.co" // Client reads the registry. type Client struct { base string hc *http.Client } // New builds a client against an endpoint, or the public one when empty. func New(endpoint string, timeout time.Duration) *Client { if endpoint == "" { endpoint = DefaultEndpoint } if timeout <= 0 { // Short. This runs while somebody is looking at a page, and a registry // that is slow today is better reported than waited for. timeout = 10 * time.Second } return &Client{base: strings.TrimSuffix(endpoint, "/"), hc: &http.Client{Timeout: timeout}} } // Model is a repository as a search result shows it. type Model struct { ID string `json:"id"` Downloads int64 `json:"downloads"` Likes int64 `json:"likes"` PipelineTag string `json:"pipeline_tag"` Tags []string `json:"tags"` LastModified time.Time `json:"lastModified"` // Gated is true when the repository asks for terms to be accepted. It // decodes from either a bool or a string ("auto", "manual"), which is why // it is not a plain bool here. Gated Gated `json:"gated"` Private bool `json:"private"` } // Gated is Hugging Face's gating flag, which is false or a word. type Gated struct{ Reason string } // UnmarshalJSON accepts false, true, or a string. // // The field is documented as a boolean and is a string in practice for most // gated repositories. Decoding it as a bool fails the whole response, which // would make one gated model in a page of results hide the other nineteen. func (g *Gated) UnmarshalJSON(b []byte) error { var s string if err := json.Unmarshal(b, &s); err == nil { g.Reason = s return nil } var v bool if err := json.Unmarshal(b, &v); err != nil { return nil // unknown shape: treat as not gated rather than fail the page } if v { g.Reason = "yes" } return nil } // Is reports whether the repository is gated at all. func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" } // Search finds repositories by name. func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) { if strings.TrimSpace(query) == "" { return nil, fmt.Errorf("search for something") } if limit <= 0 || limit > 50 { limit = 20 } q := url.Values{ "search": {query}, "limit": {fmt.Sprint(limit)}, // By downloads, because the thing somebody means by a short name is // almost always the one everybody else meant too. "sort": {"downloads"}, "direction": {"-1"}, } var out []Model err := c.get(ctx, "/api/models?"+q.Encode(), &out) return out, err } // File is one file in a repository. type File struct { Path string // Bytes is the real size. For a file stored in LFS — every weights file — // the plain size field is the size of the pointer, a few hundred bytes, so // reading that one would report a 5 GB model as 133 bytes. Bytes int64 // SHA256 is what the registry publishes for an LFS file, so it can be // compared with what the mirror computes after fetching. Empty for a small // file stored in git, where the oid is a git blob hash and not a checksum // of the content. SHA256 string } // IsWeights reports whether this file is a model rather than documentation. // // By extension, which is crude and right: the registry does not label them, and // an operator filling a mirror wants the four files out of thirty that are // worth gigabytes. func (f File) IsWeights() bool { for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} { if strings.HasSuffix(strings.ToLower(f.Path), ext) { return true } } return false } // Repo is a repository's current revision and what is in it. type Repo struct { ID string // Revision is the commit the listing came from. // // Carried so a mirrored copy can name it. Fetching from "main" records // nothing about what was fetched, and a repository that is re-uploaded // under the same branch leaves two different sets of weights with the same // provenance and no way to tell them apart. Revision string Gated Gated Files []File } // ShortRevision is the revision as a person would write it. func (r Repo) ShortRevision() string { if len(r.Revision) > 8 { return r.Revision[:8] } return r.Revision } // Repo reads a repository's revision and file list. func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { id = strings.Trim(strings.TrimSpace(id), "/") if !strings.Contains(id, "/") { return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id) } var meta struct { ID string `json:"id"` SHA string `json:"sha"` Gated Gated `json:"gated"` } if err := c.get(ctx, "/api/models/"+id, &meta); err != nil { return Repo{}, err } var tree []struct { Type string `json:"type"` Path string `json:"path"` Size int64 `json:"size"` LFS *struct { OID string `json:"oid"` Size int64 `json:"size"` } `json:"lfs"` } // By revision rather than by branch, so the listing and the URLs built from // it describe the same commit even if the branch moves in between. rev := meta.SHA if rev == "" { rev = "main" } if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil { return Repo{}, err } r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated} if r.ID == "" { r.ID = id } for _, e := range tree { if e.Type != "file" { continue } f := File{Path: e.Path, Bytes: e.Size} if e.LFS != nil { f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID } r.Files = append(r.Files, f) } return r, nil } // DownloadURL is where the mirror should fetch a file from. // // Pinned to the revision the listing came from. "main" would mean the mirror // fetches whatever is there when it gets round to it, which is not necessarily // what was on screen when somebody pressed the button. func (r Repo) DownloadURL(base, path string) string { if base == "" { base = DefaultEndpoint } rev := r.Revision if rev == "" { rev = "main" } return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path } func (c *Client) get(ctx context.Context, path string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil) if err != nil { return err } resp, err := c.hc.Do(req) if err != nil { return fmt.Errorf("reaching the model registry: %w", err) } defer resp.Body.Close() // Capped: this is a third party, and a page that renders what it sends // should not be handed a gigabyte of it. body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) if err != nil { return err } switch { case resp.StatusCode == http.StatusNotFound: return fmt.Errorf("the registry has nothing at %s", path) case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden: // Said plainly, because the next thing somebody does is check their // network. It is the repository, not the connection. return fmt.Errorf("the registry refused this anonymously; the repository is gated or private") case resp.StatusCode >= 300: return fmt.Errorf("the registry answered %d", resp.StatusCode) } if err := json.Unmarshal(body, out); err != nil { return fmt.Errorf("the registry's answer was not the JSON expected: %w", err) } return nil } // shardPattern matches the "-00002-of-00004" a split file carries before its // extension. Five digits in practice, but not required: the count is what // matters and a repository that used three would otherwise list every part as // a separate model. var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`) // FileSet is one thing worth fetching: a file, or every shard of one. // // The distinction the page turns on. Two GGUF files in a repository are usually // alternatives — q4_k_m or q8_0, you want one — while two safetensors files are // halves of the same weights and useless apart. Fetching "all the weights" is // right for the second and wasteful for the first, so neither is offered: // a set is offered, and a set is whole. type FileSet struct { // Name is what to call it. For shards it keeps the -of- notation, because // a repository can hold both the split and the joined copy of the same // quantisation and they must not read as one entry. Name string Files []File Bytes int64 Shards int } // SHA256 is the checksum when there is one to give. // // Empty for a shard set: several files have several checksums and no single // value for the set, and showing one of them invites comparing it against the // wrong thing. func (s FileSet) SHA256() string { if len(s.Files) == 1 { return s.Files[0].SHA256 } return "" } // Group collects shards into sets, leaving every other file on its own. // // Order is preserved from the listing, so a repository that sorts its files // sensibly still reads sensibly here. func Group(files []File) []FileSet { var out []FileSet at := map[string]int{} for _, f := range files { m := shardPattern.FindStringSubmatch(f.Path) if m == nil { out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes}) continue } prefix, total, ext := m[1], m[3], m[4] name := prefix + "-*-of-" + total + ext i, ok := at[name] if !ok { at[name] = len(out) out = append(out, FileSet{Name: name}) i = at[name] } out[i].Files = append(out[i].Files, f) out[i].Bytes += f.Bytes out[i].Shards = len(out[i].Files) } return out }