Files
llmgw/internal/hf/hf.go
T
Ilker ManapandClaude Opus 5 a588646b7c
CI / test (push) Successful in 1m39s
Fetch a sharded model in one action, not one shard at a time
The explorer listed every weight file with its own button. For a repository of
alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded
model it is fifteen clicks to fetch one thing, with nothing stopping you from
losing count and leaving a version with fourteen of its parts.

Files are grouped into sets now. A run of "-00002-of-00004" is one row and one
button; everything else stays its own row.

Not "fetch all the weights", which would be wrong half the time: two safetensors
files are halves of one thing and useless apart, two GGUF files are usually
alternatives and fetching both spends gigabytes on a copy nobody asked for. The
set is the unit that is whole.

A repository can hold both the split and the joined copy of the same
quantisation — the real Qwen coder repos do — so a shard set keeps the -of-
notation in its name rather than collapsing onto the name of the whole file
beside it.

A set publishes no single checksum. Several files have several, and showing one
of them invites comparing it against the wrong thing.

The form posts every source in the set, and a failure part way says how many
already started: a set that fails on its fourth shard has three arriving, and
reading that as "nothing happened" sends somebody to fetch them all again.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 15:32:27 +02:00

324 lines
10 KiB
Go

// Package hf reads the Hugging Face model registry.
//
// Enough of it to answer two questions an operator has before mirroring a
// model: what is there, and which file do I want. Nothing here downloads
// anything — the mirror does that, over its own network, and the point of the
// split is that the control plane never carries a model's bytes.
//
// Read-only and anonymous. A gated or private repository answers 401 to this
// client, which is reported rather than worked around: a token belongs to the
// machine that fetches the weights, not to the one browsing a catalogue.
package hf
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
)
// DefaultEndpoint is the public registry.
const DefaultEndpoint = "https://huggingface.co"
// Client reads the registry.
type Client struct {
base string
hc *http.Client
}
// New builds a client against an endpoint, or the public one when empty.
func New(endpoint string, timeout time.Duration) *Client {
if endpoint == "" {
endpoint = DefaultEndpoint
}
if timeout <= 0 {
// Short. This runs while somebody is looking at a page, and a registry
// that is slow today is better reported than waited for.
timeout = 10 * time.Second
}
return &Client{base: strings.TrimSuffix(endpoint, "/"),
hc: &http.Client{Timeout: timeout}}
}
// Model is a repository as a search result shows it.
type Model struct {
ID string `json:"id"`
Downloads int64 `json:"downloads"`
Likes int64 `json:"likes"`
PipelineTag string `json:"pipeline_tag"`
Tags []string `json:"tags"`
LastModified time.Time `json:"lastModified"`
// Gated is true when the repository asks for terms to be accepted. It
// decodes from either a bool or a string ("auto", "manual"), which is why
// it is not a plain bool here.
Gated Gated `json:"gated"`
Private bool `json:"private"`
}
// Gated is Hugging Face's gating flag, which is false or a word.
type Gated struct{ Reason string }
// UnmarshalJSON accepts false, true, or a string.
//
// The field is documented as a boolean and is a string in practice for most
// gated repositories. Decoding it as a bool fails the whole response, which
// would make one gated model in a page of results hide the other nineteen.
func (g *Gated) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err == nil {
g.Reason = s
return nil
}
var v bool
if err := json.Unmarshal(b, &v); err != nil {
return nil // unknown shape: treat as not gated rather than fail the page
}
if v {
g.Reason = "yes"
}
return nil
}
// Is reports whether the repository is gated at all.
func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" }
// Search finds repositories by name.
func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) {
if strings.TrimSpace(query) == "" {
return nil, fmt.Errorf("search for something")
}
if limit <= 0 || limit > 50 {
limit = 20
}
q := url.Values{
"search": {query},
"limit": {fmt.Sprint(limit)},
// By downloads, because the thing somebody means by a short name is
// almost always the one everybody else meant too.
"sort": {"downloads"},
"direction": {"-1"},
}
var out []Model
err := c.get(ctx, "/api/models?"+q.Encode(), &out)
return out, err
}
// File is one file in a repository.
type File struct {
Path string
// Bytes is the real size. For a file stored in LFS — every weights file —
// the plain size field is the size of the pointer, a few hundred bytes, so
// reading that one would report a 5 GB model as 133 bytes.
Bytes int64
// SHA256 is what the registry publishes for an LFS file, so it can be
// compared with what the mirror computes after fetching. Empty for a small
// file stored in git, where the oid is a git blob hash and not a checksum
// of the content.
SHA256 string
}
// IsWeights reports whether this file is a model rather than documentation.
//
// By extension, which is crude and right: the registry does not label them, and
// an operator filling a mirror wants the four files out of thirty that are
// worth gigabytes.
func (f File) IsWeights() bool {
for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} {
if strings.HasSuffix(strings.ToLower(f.Path), ext) {
return true
}
}
return false
}
// Repo is a repository's current revision and what is in it.
type Repo struct {
ID string
// Revision is the commit the listing came from.
//
// Carried so a mirrored copy can name it. Fetching from "main" records
// nothing about what was fetched, and a repository that is re-uploaded
// under the same branch leaves two different sets of weights with the same
// provenance and no way to tell them apart.
Revision string
Gated Gated
Files []File
}
// ShortRevision is the revision as a person would write it.
func (r Repo) ShortRevision() string {
if len(r.Revision) > 8 {
return r.Revision[:8]
}
return r.Revision
}
// Repo reads a repository's revision and file list.
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
id = strings.Trim(strings.TrimSpace(id), "/")
if !strings.Contains(id, "/") {
return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id)
}
var meta struct {
ID string `json:"id"`
SHA string `json:"sha"`
Gated Gated `json:"gated"`
}
if err := c.get(ctx, "/api/models/"+id, &meta); err != nil {
return Repo{}, err
}
var tree []struct {
Type string `json:"type"`
Path string `json:"path"`
Size int64 `json:"size"`
LFS *struct {
OID string `json:"oid"`
Size int64 `json:"size"`
} `json:"lfs"`
}
// By revision rather than by branch, so the listing and the URLs built from
// it describe the same commit even if the branch moves in between.
rev := meta.SHA
if rev == "" {
rev = "main"
}
if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil {
return Repo{}, err
}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
if r.ID == "" {
r.ID = id
}
for _, e := range tree {
if e.Type != "file" {
continue
}
f := File{Path: e.Path, Bytes: e.Size}
if e.LFS != nil {
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
}
r.Files = append(r.Files, f)
}
return r, nil
}
// DownloadURL is where the mirror should fetch a file from.
//
// Pinned to the revision the listing came from. "main" would mean the mirror
// fetches whatever is there when it gets round to it, which is not necessarily
// what was on screen when somebody pressed the button.
func (r Repo) DownloadURL(base, path string) string {
if base == "" {
base = DefaultEndpoint
}
rev := r.Revision
if rev == "" {
rev = "main"
}
return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path
}
func (c *Client) get(ctx context.Context, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return fmt.Errorf("reaching the model registry: %w", err)
}
defer resp.Body.Close()
// Capped: this is a third party, and a page that renders what it sends
// should not be handed a gigabyte of it.
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return err
}
switch {
case resp.StatusCode == http.StatusNotFound:
return fmt.Errorf("the registry has nothing at %s", path)
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
// Said plainly, because the next thing somebody does is check their
// network. It is the repository, not the connection.
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
case resp.StatusCode >= 300:
return fmt.Errorf("the registry answered %d", resp.StatusCode)
}
if err := json.Unmarshal(body, out); err != nil {
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
}
return nil
}
// shardPattern matches the "-00002-of-00004" a split file carries before its
// extension. Five digits in practice, but not required: the count is what
// matters and a repository that used three would otherwise list every part as
// a separate model.
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
// FileSet is one thing worth fetching: a file, or every shard of one.
//
// The distinction the page turns on. Two GGUF files in a repository are usually
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
// halves of the same weights and useless apart. Fetching "all the weights" is
// right for the second and wasteful for the first, so neither is offered:
// a set is offered, and a set is whole.
type FileSet struct {
// Name is what to call it. For shards it keeps the -of- notation, because
// a repository can hold both the split and the joined copy of the same
// quantisation and they must not read as one entry.
Name string
Files []File
Bytes int64
Shards int
}
// SHA256 is the checksum when there is one to give.
//
// Empty for a shard set: several files have several checksums and no single
// value for the set, and showing one of them invites comparing it against the
// wrong thing.
func (s FileSet) SHA256() string {
if len(s.Files) == 1 {
return s.Files[0].SHA256
}
return ""
}
// Group collects shards into sets, leaving every other file on its own.
//
// Order is preserved from the listing, so a repository that sorts its files
// sensibly still reads sensibly here.
func Group(files []File) []FileSet {
var out []FileSet
at := map[string]int{}
for _, f := range files {
m := shardPattern.FindStringSubmatch(f.Path)
if m == nil {
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
continue
}
prefix, total, ext := m[1], m[3], m[4]
name := prefix + "-*-of-" + total + ext
i, ok := at[name]
if !ok {
at[name] = len(out)
out = append(out, FileSet{Name: name})
i = at[name]
}
out[i].Files = append(out[i].Files, f)
out[i].Bytes += f.Bytes
out[i].Shards = len(out[i].Files)
}
return out
}