Files
llmgw/internal/hf/hf.go
T
Ilker ManapandClaude Opus 5.5 02fa639657 Open the mirror's registry on what is current, not an empty search box
The mirror page lists Hugging Face's trending models without anything
typed, with views for what is new this month and what is most
downloaded, filters for task and size, and each repository's parameter
count, architecture, context length and age from one registry call.
"Only what these nodes can run" is on by default and leaves out
architectures the nodes' llama.cpp cannot load, and adapters, saying how
many it hid. Listings are remembered for ten minutes.

The stylesheet is now linked by a hash of its content. It is cached for
a day, so under a fixed name a deploy's new styles reached people a day
after its markup.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 21:56:58 +02:00

451 lines
14 KiB
Go

// Package hf reads the Hugging Face model registry.
//
// Enough of it to answer two questions an operator has before mirroring a
// model: what is there, and which file do I want. Nothing here downloads
// anything — the mirror does that, over its own network, and the point of the
// split is that the control plane never carries a model's bytes.
//
// Read-only and anonymous. A gated or private repository answers 401 to this
// client, which is reported rather than worked around: a token belongs to the
// machine that fetches the weights, not to the one browsing a catalogue.
package hf
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
)
// DefaultEndpoint is the public registry.
const DefaultEndpoint = "https://huggingface.co"
// Client reads the registry.
type Client struct {
base string
hc *http.Client
mu sync.Mutex
cache map[string]cached
}
// New builds a client against an endpoint, or the public one when empty.
func New(endpoint string, timeout time.Duration) *Client {
if endpoint == "" {
endpoint = DefaultEndpoint
}
if timeout <= 0 {
// Short. This runs while somebody is looking at a page, and a registry
// that is slow today is better reported than waited for.
timeout = 10 * time.Second
}
return &Client{base: strings.TrimSuffix(endpoint, "/"),
hc: &http.Client{Timeout: timeout}}
}
// Model is a repository as a search result shows it.
type Model struct {
ID string `json:"id"`
Downloads int64 `json:"downloads"`
Likes int64 `json:"likes"`
PipelineTag string `json:"pipeline_tag"`
Tags []string `json:"tags"`
LastModified time.Time `json:"lastModified"`
// Gated is true when the repository asks for terms to be accepted. It
// decodes from either a bool or a string ("auto", "manual"), which is why
// it is not a plain bool here.
Gated Gated `json:"gated"`
Private bool `json:"private"`
CreatedAt time.Time `json:"createdAt"`
TrendingScore float64 `json:"trendingScore"`
// GGUF is what the registry read from the repository's weights, absent
// for one that has none.
GGUF *GGUFInfo `json:"gguf"`
}
// GGUFInfo is the registry's summary of a repository's GGUF weights.
type GGUFInfo struct {
// Total is the parameter count.
Total int64 `json:"total"`
Architecture string `json:"architecture"`
ContextLength int64 `json:"context_length"`
}
// Params is the parameter count, 0 when the registry does not know it.
func (m Model) Params() int64 {
if m.GGUF == nil {
return 0
}
return m.GGUF.Total
}
// Architecture is the GGUF architecture, empty when unknown.
func (m Model) Architecture() string {
if m.GGUF == nil {
return ""
}
return m.GGUF.Architecture
}
// Gated is Hugging Face's gating flag, which is false or a word.
type Gated struct{ Reason string }
// UnmarshalJSON accepts false, true, or a string.
//
// The field is documented as a boolean and is a string in practice for most
// gated repositories. Decoding it as a bool fails the whole response, which
// would make one gated model in a page of results hide the other nineteen.
func (g *Gated) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err == nil {
g.Reason = s
return nil
}
var v bool
if err := json.Unmarshal(b, &v); err != nil {
return nil // unknown shape: treat as not gated rather than fail the page
}
if v {
g.Reason = "yes"
}
return nil
}
// Is reports whether the repository is gated at all.
func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" }
// Search finds repositories by name.
func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) {
if strings.TrimSpace(query) == "" {
return nil, fmt.Errorf("search for something")
}
// By downloads, because the thing somebody means by a short name is
// almost always the one everybody else meant too.
return c.Explore(ctx, Query{Search: query, Sort: SortDownloads, Limit: limit})
}
// Sort orders an exploration.
type Sort string
// The orders the registry offers that are worth offering.
const (
SortTrending Sort = "trendingScore"
SortDownloads Sort = "downloads"
SortLikes Sort = "likes"
SortCreated Sort = "createdAt"
)
// Query is what to list from the registry.
type Query struct {
Search string
Sort Sort
// Task is a pipeline tag: text-generation, feature-extraction. Empty is
// any.
Task string
// GGUF keeps to repositories with GGUF weights, which is what llama.cpp
// loads.
GGUF bool
// MaxParams bounds the parameter count; 0 is no bound. The registry
// applies it, so a page of small models is a full page.
MaxParams int64
Limit int
}
// exploreFields is everything a listing shows. Asking for any field with
// expand returns only the fields asked for, so all of them are named.
var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified",
"createdAt", "gated", "private", "trendingScore", "gguf"}
// Explore lists repositories: the trending ones, the newest, or a search.
//
// Remembered for a few minutes. The mirror page opens on this, and a page
// that waits on a registry on the far side of the internet every time it is
// opened is slow for news that changes by the hour.
func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) {
if q.Limit <= 0 || q.Limit > 100 {
q.Limit = 20
}
if q.Sort == "" {
q.Sort = SortTrending
}
v := url.Values{
"limit": {fmt.Sprint(q.Limit)},
"sort": {string(q.Sort)},
"direction": {"-1"},
"expand[]": exploreFields,
}
if s := strings.TrimSpace(q.Search); s != "" {
v.Set("search", s)
}
if q.Task != "" {
v.Set("pipeline_tag", q.Task)
}
if q.GGUF {
v.Set("filter", "gguf")
}
if q.MaxParams > 0 {
v.Set("num_parameters", fmt.Sprintf("min:0,max:%d", q.MaxParams))
}
path := "/api/models?" + v.Encode()
c.mu.Lock()
if e, ok := c.cache[path]; ok && time.Since(e.at) < exploreTTL {
c.mu.Unlock()
return e.models, nil
}
c.mu.Unlock()
var out []Model
if err := c.get(ctx, path, &out); err != nil {
return nil, err
}
c.mu.Lock()
if c.cache == nil || len(c.cache) > 200 {
c.cache = map[string]cached{}
}
c.cache[path] = cached{at: time.Now(), models: out}
c.mu.Unlock()
return out, nil
}
// exploreTTL is how long a listing is reused.
const exploreTTL = 10 * time.Minute
type cached struct {
at time.Time
models []Model
}
// File is one file in a repository.
type File struct {
Path string
// Bytes is the real size. For a file stored in LFS — every weights file —
// the plain size field is the size of the pointer, a few hundred bytes, so
// reading that one would report a 5 GB model as 133 bytes.
Bytes int64
// SHA256 is what the registry publishes for an LFS file, so it can be
// compared with what the mirror computes after fetching. Empty for a small
// file stored in git, where the oid is a git blob hash and not a checksum
// of the content.
SHA256 string
}
// IsWeights reports whether this file is a model rather than documentation.
//
// By extension, which is crude and right: the registry does not label them, and
// an operator filling a mirror wants the four files out of thirty that are
// worth gigabytes.
func (f File) IsWeights() bool {
for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} {
if strings.HasSuffix(strings.ToLower(f.Path), ext) {
return true
}
}
return false
}
// Repo is a repository's current revision and what is in it.
type Repo struct {
ID string
// Revision is the commit the listing came from.
//
// Carried so a mirrored copy can name it. Fetching from "main" records
// nothing about what was fetched, and a repository that is re-uploaded
// under the same branch leaves two different sets of weights with the same
// provenance and no way to tell them apart.
Revision string
Gated Gated
Files []File
// Architecture is the GGUF architecture the registry read from the
// repository's weights, empty for a repository with none. What decides
// whether llama.cpp can load it, known before anything is downloaded.
Architecture string
}
// ShortRevision is the revision as a person would write it.
func (r Repo) ShortRevision() string {
if len(r.Revision) > 8 {
return r.Revision[:8]
}
return r.Revision
}
// Repo reads a repository's revision and file list.
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
id = strings.Trim(strings.TrimSpace(id), "/")
if !strings.Contains(id, "/") {
return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id)
}
var meta struct {
ID string `json:"id"`
SHA string `json:"sha"`
Gated Gated `json:"gated"`
GGUF *struct {
Architecture string `json:"architecture"`
} `json:"gguf"`
}
// Expanded to what this reads: asking for gguf alone would leave out the
// revision and the gate.
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
return Repo{}, err
}
var tree []struct {
Type string `json:"type"`
Path string `json:"path"`
Size int64 `json:"size"`
LFS *struct {
OID string `json:"oid"`
Size int64 `json:"size"`
} `json:"lfs"`
}
// By revision rather than by branch, so the listing and the URLs built from
// it describe the same commit even if the branch moves in between.
rev := meta.SHA
if rev == "" {
rev = "main"
}
if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil {
return Repo{}, err
}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
if meta.GGUF != nil {
r.Architecture = meta.GGUF.Architecture
}
if r.ID == "" {
r.ID = id
}
for _, e := range tree {
if e.Type != "file" {
continue
}
f := File{Path: e.Path, Bytes: e.Size}
if e.LFS != nil {
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
}
r.Files = append(r.Files, f)
}
return r, nil
}
// DownloadURL is where the mirror should fetch a file from.
//
// Pinned to the revision the listing came from. "main" would mean the mirror
// fetches whatever is there when it gets round to it, which is not necessarily
// what was on screen when somebody pressed the button.
func (r Repo) DownloadURL(base, path string) string {
if base == "" {
base = DefaultEndpoint
}
rev := r.Revision
if rev == "" {
rev = "main"
}
return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path
}
func (c *Client) get(ctx context.Context, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return fmt.Errorf("reaching the model registry: %w", err)
}
defer resp.Body.Close()
// Capped: this is a third party, and a page that renders what it sends
// should not be handed a gigabyte of it.
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return err
}
switch {
case resp.StatusCode == http.StatusNotFound:
return fmt.Errorf("the registry has nothing at %s", path)
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
// Said plainly, because the next thing somebody does is check their
// network. It is the repository, not the connection.
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
case resp.StatusCode >= 300:
return fmt.Errorf("the registry answered %d", resp.StatusCode)
}
if err := json.Unmarshal(body, out); err != nil {
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
}
return nil
}
// shardPattern matches the "-00002-of-00004" a split file carries before its
// extension. Five digits in practice, but not required: the count is what
// matters and a repository that used three would otherwise list every part as
// a separate model.
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
// FileSet is one thing worth fetching: a file, or every shard of one.
//
// The distinction the page turns on. Two GGUF files in a repository are usually
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
// halves of the same weights and useless apart. Fetching "all the weights" is
// right for the second and wasteful for the first, so neither is offered:
// a set is offered, and a set is whole.
type FileSet struct {
// Name is what to call it. For shards it keeps the -of- notation, because
// a repository can hold both the split and the joined copy of the same
// quantisation and they must not read as one entry.
Name string
Files []File
Bytes int64
Shards int
}
// SHA256 is the checksum when there is one to give.
//
// Empty for a shard set: several files have several checksums and no single
// value for the set, and showing one of them invites comparing it against the
// wrong thing.
func (s FileSet) SHA256() string {
if len(s.Files) == 1 {
return s.Files[0].SHA256
}
return ""
}
// Group collects shards into sets, leaving every other file on its own.
//
// Order is preserved from the listing, so a repository that sorts its files
// sensibly still reads sensibly here.
func Group(files []File) []FileSet {
var out []FileSet
at := map[string]int{}
for _, f := range files {
m := shardPattern.FindStringSubmatch(f.Path)
if m == nil {
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
continue
}
prefix, total, ext := m[1], m[3], m[4]
name := prefix + "-*-of-" + total + ext
i, ok := at[name]
if !ok {
at[name] = len(out)
out = append(out, FileSet{Name: name})
i = at[name]
}
out[i].Files = append(out[i].Files, f)
out[i].Bytes += f.Bytes
out[i].Shards = len(out[i].Files)
}
return out
}