CI / test (push) Successful in 1m39s
The explorer listed every weight file with its own button. For a repository of alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded model it is fifteen clicks to fetch one thing, with nothing stopping you from losing count and leaving a version with fourteen of its parts. Files are grouped into sets now. A run of "-00002-of-00004" is one row and one button; everything else stays its own row. Not "fetch all the weights", which would be wrong half the time: two safetensors files are halves of one thing and useless apart, two GGUF files are usually alternatives and fetching both spends gigabytes on a copy nobody asked for. The set is the unit that is whole. A repository can hold both the split and the joined copy of the same quantisation — the real Qwen coder repos do — so a shard set keeps the -of- notation in its name rather than collapsing onto the name of the whole file beside it. A set publishes no single checksum. Several files have several, and showing one of them invites comparing it against the wrong thing. The form posts every source in the set, and a failure part way says how many already started: a set that fails on its fourth shard has three arriving, and reading that as "nothing happened" sends somebody to fetch them all again. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
324 lines
10 KiB
Go
324 lines
10 KiB
Go
// Package hf reads the Hugging Face model registry.
|
|
//
|
|
// Enough of it to answer two questions an operator has before mirroring a
|
|
// model: what is there, and which file do I want. Nothing here downloads
|
|
// anything — the mirror does that, over its own network, and the point of the
|
|
// split is that the control plane never carries a model's bytes.
|
|
//
|
|
// Read-only and anonymous. A gated or private repository answers 401 to this
|
|
// client, which is reported rather than worked around: a token belongs to the
|
|
// machine that fetches the weights, not to the one browsing a catalogue.
|
|
package hf
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// DefaultEndpoint is the public registry.
|
|
const DefaultEndpoint = "https://huggingface.co"
|
|
|
|
// Client reads the registry.
|
|
type Client struct {
|
|
base string
|
|
hc *http.Client
|
|
}
|
|
|
|
// New builds a client against an endpoint, or the public one when empty.
|
|
func New(endpoint string, timeout time.Duration) *Client {
|
|
if endpoint == "" {
|
|
endpoint = DefaultEndpoint
|
|
}
|
|
if timeout <= 0 {
|
|
// Short. This runs while somebody is looking at a page, and a registry
|
|
// that is slow today is better reported than waited for.
|
|
timeout = 10 * time.Second
|
|
}
|
|
return &Client{base: strings.TrimSuffix(endpoint, "/"),
|
|
hc: &http.Client{Timeout: timeout}}
|
|
}
|
|
|
|
// Model is a repository as a search result shows it.
|
|
type Model struct {
|
|
ID string `json:"id"`
|
|
Downloads int64 `json:"downloads"`
|
|
Likes int64 `json:"likes"`
|
|
PipelineTag string `json:"pipeline_tag"`
|
|
Tags []string `json:"tags"`
|
|
LastModified time.Time `json:"lastModified"`
|
|
// Gated is true when the repository asks for terms to be accepted. It
|
|
// decodes from either a bool or a string ("auto", "manual"), which is why
|
|
// it is not a plain bool here.
|
|
Gated Gated `json:"gated"`
|
|
Private bool `json:"private"`
|
|
}
|
|
|
|
// Gated is Hugging Face's gating flag, which is false or a word.
|
|
type Gated struct{ Reason string }
|
|
|
|
// UnmarshalJSON accepts false, true, or a string.
|
|
//
|
|
// The field is documented as a boolean and is a string in practice for most
|
|
// gated repositories. Decoding it as a bool fails the whole response, which
|
|
// would make one gated model in a page of results hide the other nineteen.
|
|
func (g *Gated) UnmarshalJSON(b []byte) error {
|
|
var s string
|
|
if err := json.Unmarshal(b, &s); err == nil {
|
|
g.Reason = s
|
|
return nil
|
|
}
|
|
var v bool
|
|
if err := json.Unmarshal(b, &v); err != nil {
|
|
return nil // unknown shape: treat as not gated rather than fail the page
|
|
}
|
|
if v {
|
|
g.Reason = "yes"
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Is reports whether the repository is gated at all.
|
|
func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" }
|
|
|
|
// Search finds repositories by name.
|
|
func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) {
|
|
if strings.TrimSpace(query) == "" {
|
|
return nil, fmt.Errorf("search for something")
|
|
}
|
|
if limit <= 0 || limit > 50 {
|
|
limit = 20
|
|
}
|
|
q := url.Values{
|
|
"search": {query},
|
|
"limit": {fmt.Sprint(limit)},
|
|
// By downloads, because the thing somebody means by a short name is
|
|
// almost always the one everybody else meant too.
|
|
"sort": {"downloads"},
|
|
"direction": {"-1"},
|
|
}
|
|
var out []Model
|
|
err := c.get(ctx, "/api/models?"+q.Encode(), &out)
|
|
return out, err
|
|
}
|
|
|
|
// File is one file in a repository.
|
|
type File struct {
|
|
Path string
|
|
// Bytes is the real size. For a file stored in LFS — every weights file —
|
|
// the plain size field is the size of the pointer, a few hundred bytes, so
|
|
// reading that one would report a 5 GB model as 133 bytes.
|
|
Bytes int64
|
|
// SHA256 is what the registry publishes for an LFS file, so it can be
|
|
// compared with what the mirror computes after fetching. Empty for a small
|
|
// file stored in git, where the oid is a git blob hash and not a checksum
|
|
// of the content.
|
|
SHA256 string
|
|
}
|
|
|
|
// IsWeights reports whether this file is a model rather than documentation.
|
|
//
|
|
// By extension, which is crude and right: the registry does not label them, and
|
|
// an operator filling a mirror wants the four files out of thirty that are
|
|
// worth gigabytes.
|
|
func (f File) IsWeights() bool {
|
|
for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} {
|
|
if strings.HasSuffix(strings.ToLower(f.Path), ext) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Repo is a repository's current revision and what is in it.
|
|
type Repo struct {
|
|
ID string
|
|
// Revision is the commit the listing came from.
|
|
//
|
|
// Carried so a mirrored copy can name it. Fetching from "main" records
|
|
// nothing about what was fetched, and a repository that is re-uploaded
|
|
// under the same branch leaves two different sets of weights with the same
|
|
// provenance and no way to tell them apart.
|
|
Revision string
|
|
Gated Gated
|
|
Files []File
|
|
}
|
|
|
|
// ShortRevision is the revision as a person would write it.
|
|
func (r Repo) ShortRevision() string {
|
|
if len(r.Revision) > 8 {
|
|
return r.Revision[:8]
|
|
}
|
|
return r.Revision
|
|
}
|
|
|
|
// Repo reads a repository's revision and file list.
|
|
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
|
id = strings.Trim(strings.TrimSpace(id), "/")
|
|
if !strings.Contains(id, "/") {
|
|
return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id)
|
|
}
|
|
|
|
var meta struct {
|
|
ID string `json:"id"`
|
|
SHA string `json:"sha"`
|
|
Gated Gated `json:"gated"`
|
|
}
|
|
if err := c.get(ctx, "/api/models/"+id, &meta); err != nil {
|
|
return Repo{}, err
|
|
}
|
|
|
|
var tree []struct {
|
|
Type string `json:"type"`
|
|
Path string `json:"path"`
|
|
Size int64 `json:"size"`
|
|
LFS *struct {
|
|
OID string `json:"oid"`
|
|
Size int64 `json:"size"`
|
|
} `json:"lfs"`
|
|
}
|
|
// By revision rather than by branch, so the listing and the URLs built from
|
|
// it describe the same commit even if the branch moves in between.
|
|
rev := meta.SHA
|
|
if rev == "" {
|
|
rev = "main"
|
|
}
|
|
if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil {
|
|
return Repo{}, err
|
|
}
|
|
|
|
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
|
|
if r.ID == "" {
|
|
r.ID = id
|
|
}
|
|
for _, e := range tree {
|
|
if e.Type != "file" {
|
|
continue
|
|
}
|
|
f := File{Path: e.Path, Bytes: e.Size}
|
|
if e.LFS != nil {
|
|
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
|
|
}
|
|
r.Files = append(r.Files, f)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// DownloadURL is where the mirror should fetch a file from.
|
|
//
|
|
// Pinned to the revision the listing came from. "main" would mean the mirror
|
|
// fetches whatever is there when it gets round to it, which is not necessarily
|
|
// what was on screen when somebody pressed the button.
|
|
func (r Repo) DownloadURL(base, path string) string {
|
|
if base == "" {
|
|
base = DefaultEndpoint
|
|
}
|
|
rev := r.Revision
|
|
if rev == "" {
|
|
rev = "main"
|
|
}
|
|
return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path
|
|
}
|
|
|
|
func (c *Client) get(ctx context.Context, path string, out any) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := c.hc.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("reaching the model registry: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Capped: this is a third party, and a page that renders what it sends
|
|
// should not be handed a gigabyte of it.
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch {
|
|
case resp.StatusCode == http.StatusNotFound:
|
|
return fmt.Errorf("the registry has nothing at %s", path)
|
|
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
|
|
// Said plainly, because the next thing somebody does is check their
|
|
// network. It is the repository, not the connection.
|
|
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
|
|
case resp.StatusCode >= 300:
|
|
return fmt.Errorf("the registry answered %d", resp.StatusCode)
|
|
}
|
|
if err := json.Unmarshal(body, out); err != nil {
|
|
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// shardPattern matches the "-00002-of-00004" a split file carries before its
|
|
// extension. Five digits in practice, but not required: the count is what
|
|
// matters and a repository that used three would otherwise list every part as
|
|
// a separate model.
|
|
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
|
|
|
|
// FileSet is one thing worth fetching: a file, or every shard of one.
|
|
//
|
|
// The distinction the page turns on. Two GGUF files in a repository are usually
|
|
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
|
|
// halves of the same weights and useless apart. Fetching "all the weights" is
|
|
// right for the second and wasteful for the first, so neither is offered:
|
|
// a set is offered, and a set is whole.
|
|
type FileSet struct {
|
|
// Name is what to call it. For shards it keeps the -of- notation, because
|
|
// a repository can hold both the split and the joined copy of the same
|
|
// quantisation and they must not read as one entry.
|
|
Name string
|
|
Files []File
|
|
Bytes int64
|
|
Shards int
|
|
}
|
|
|
|
// SHA256 is the checksum when there is one to give.
|
|
//
|
|
// Empty for a shard set: several files have several checksums and no single
|
|
// value for the set, and showing one of them invites comparing it against the
|
|
// wrong thing.
|
|
func (s FileSet) SHA256() string {
|
|
if len(s.Files) == 1 {
|
|
return s.Files[0].SHA256
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Group collects shards into sets, leaving every other file on its own.
|
|
//
|
|
// Order is preserved from the listing, so a repository that sorts its files
|
|
// sensibly still reads sensibly here.
|
|
func Group(files []File) []FileSet {
|
|
var out []FileSet
|
|
at := map[string]int{}
|
|
for _, f := range files {
|
|
m := shardPattern.FindStringSubmatch(f.Path)
|
|
if m == nil {
|
|
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
|
|
continue
|
|
}
|
|
prefix, total, ext := m[1], m[3], m[4]
|
|
name := prefix + "-*-of-" + total + ext
|
|
i, ok := at[name]
|
|
if !ok {
|
|
at[name] = len(out)
|
|
out = append(out, FileSet{Name: name})
|
|
i = at[name]
|
|
}
|
|
out[i].Files = append(out[i].Files, f)
|
|
out[i].Bytes += f.Bytes
|
|
out[i].Shards = len(out[i].Files)
|
|
}
|
|
return out
|
|
}
|