The mirror page lists Hugging Face's trending models without anything typed, with views for what is new this month and what is most downloaded, filters for task and size, and each repository's parameter count, architecture, context length and age from one registry call. "Only what these nodes can run" is on by default and leaves out architectures the nodes' llama.cpp cannot load, and adapters, saying how many it hid. Listings are remembered for ten minutes. The stylesheet is now linked by a hash of its content. It is cached for a day, so under a fixed name a deploy's new styles reached people a day after its markup. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
451 lines
14 KiB
Go
451 lines
14 KiB
Go
// Package hf reads the Hugging Face model registry.
|
|
//
|
|
// Enough of it to answer two questions an operator has before mirroring a
|
|
// model: what is there, and which file do I want. Nothing here downloads
|
|
// anything — the mirror does that, over its own network, and the point of the
|
|
// split is that the control plane never carries a model's bytes.
|
|
//
|
|
// Read-only and anonymous. A gated or private repository answers 401 to this
|
|
// client, which is reported rather than worked around: a token belongs to the
|
|
// machine that fetches the weights, not to the one browsing a catalogue.
|
|
package hf
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// DefaultEndpoint is the public registry.
|
|
const DefaultEndpoint = "https://huggingface.co"
|
|
|
|
// Client reads the registry.
|
|
type Client struct {
|
|
base string
|
|
hc *http.Client
|
|
|
|
mu sync.Mutex
|
|
cache map[string]cached
|
|
}
|
|
|
|
// New builds a client against an endpoint, or the public one when empty.
|
|
func New(endpoint string, timeout time.Duration) *Client {
|
|
if endpoint == "" {
|
|
endpoint = DefaultEndpoint
|
|
}
|
|
if timeout <= 0 {
|
|
// Short. This runs while somebody is looking at a page, and a registry
|
|
// that is slow today is better reported than waited for.
|
|
timeout = 10 * time.Second
|
|
}
|
|
return &Client{base: strings.TrimSuffix(endpoint, "/"),
|
|
hc: &http.Client{Timeout: timeout}}
|
|
}
|
|
|
|
// Model is a repository as a search result shows it.
|
|
type Model struct {
|
|
ID string `json:"id"`
|
|
Downloads int64 `json:"downloads"`
|
|
Likes int64 `json:"likes"`
|
|
PipelineTag string `json:"pipeline_tag"`
|
|
Tags []string `json:"tags"`
|
|
LastModified time.Time `json:"lastModified"`
|
|
// Gated is true when the repository asks for terms to be accepted. It
|
|
// decodes from either a bool or a string ("auto", "manual"), which is why
|
|
// it is not a plain bool here.
|
|
Gated Gated `json:"gated"`
|
|
Private bool `json:"private"`
|
|
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
TrendingScore float64 `json:"trendingScore"`
|
|
// GGUF is what the registry read from the repository's weights, absent
|
|
// for one that has none.
|
|
GGUF *GGUFInfo `json:"gguf"`
|
|
}
|
|
|
|
// GGUFInfo is the registry's summary of a repository's GGUF weights.
|
|
type GGUFInfo struct {
|
|
// Total is the parameter count.
|
|
Total int64 `json:"total"`
|
|
Architecture string `json:"architecture"`
|
|
ContextLength int64 `json:"context_length"`
|
|
}
|
|
|
|
// Params is the parameter count, 0 when the registry does not know it.
|
|
func (m Model) Params() int64 {
|
|
if m.GGUF == nil {
|
|
return 0
|
|
}
|
|
return m.GGUF.Total
|
|
}
|
|
|
|
// Architecture is the GGUF architecture, empty when unknown.
|
|
func (m Model) Architecture() string {
|
|
if m.GGUF == nil {
|
|
return ""
|
|
}
|
|
return m.GGUF.Architecture
|
|
}
|
|
|
|
// Gated is Hugging Face's gating flag, which is false or a word.
|
|
type Gated struct{ Reason string }
|
|
|
|
// UnmarshalJSON accepts false, true, or a string.
|
|
//
|
|
// The field is documented as a boolean and is a string in practice for most
|
|
// gated repositories. Decoding it as a bool fails the whole response, which
|
|
// would make one gated model in a page of results hide the other nineteen.
|
|
func (g *Gated) UnmarshalJSON(b []byte) error {
|
|
var s string
|
|
if err := json.Unmarshal(b, &s); err == nil {
|
|
g.Reason = s
|
|
return nil
|
|
}
|
|
var v bool
|
|
if err := json.Unmarshal(b, &v); err != nil {
|
|
return nil // unknown shape: treat as not gated rather than fail the page
|
|
}
|
|
if v {
|
|
g.Reason = "yes"
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Is reports whether the repository is gated at all.
|
|
func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" }
|
|
|
|
// Search finds repositories by name.
|
|
func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) {
|
|
if strings.TrimSpace(query) == "" {
|
|
return nil, fmt.Errorf("search for something")
|
|
}
|
|
// By downloads, because the thing somebody means by a short name is
|
|
// almost always the one everybody else meant too.
|
|
return c.Explore(ctx, Query{Search: query, Sort: SortDownloads, Limit: limit})
|
|
}
|
|
|
|
// Sort orders an exploration.
|
|
type Sort string
|
|
|
|
// The orders the registry offers that are worth offering.
|
|
const (
|
|
SortTrending Sort = "trendingScore"
|
|
SortDownloads Sort = "downloads"
|
|
SortLikes Sort = "likes"
|
|
SortCreated Sort = "createdAt"
|
|
)
|
|
|
|
// Query is what to list from the registry.
|
|
type Query struct {
|
|
Search string
|
|
Sort Sort
|
|
// Task is a pipeline tag: text-generation, feature-extraction. Empty is
|
|
// any.
|
|
Task string
|
|
// GGUF keeps to repositories with GGUF weights, which is what llama.cpp
|
|
// loads.
|
|
GGUF bool
|
|
// MaxParams bounds the parameter count; 0 is no bound. The registry
|
|
// applies it, so a page of small models is a full page.
|
|
MaxParams int64
|
|
Limit int
|
|
}
|
|
|
|
// exploreFields is everything a listing shows. Asking for any field with
|
|
// expand returns only the fields asked for, so all of them are named.
|
|
var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified",
|
|
"createdAt", "gated", "private", "trendingScore", "gguf"}
|
|
|
|
// Explore lists repositories: the trending ones, the newest, or a search.
|
|
//
|
|
// Remembered for a few minutes. The mirror page opens on this, and a page
|
|
// that waits on a registry on the far side of the internet every time it is
|
|
// opened is slow for news that changes by the hour.
|
|
func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) {
|
|
if q.Limit <= 0 || q.Limit > 100 {
|
|
q.Limit = 20
|
|
}
|
|
if q.Sort == "" {
|
|
q.Sort = SortTrending
|
|
}
|
|
v := url.Values{
|
|
"limit": {fmt.Sprint(q.Limit)},
|
|
"sort": {string(q.Sort)},
|
|
"direction": {"-1"},
|
|
"expand[]": exploreFields,
|
|
}
|
|
if s := strings.TrimSpace(q.Search); s != "" {
|
|
v.Set("search", s)
|
|
}
|
|
if q.Task != "" {
|
|
v.Set("pipeline_tag", q.Task)
|
|
}
|
|
if q.GGUF {
|
|
v.Set("filter", "gguf")
|
|
}
|
|
if q.MaxParams > 0 {
|
|
v.Set("num_parameters", fmt.Sprintf("min:0,max:%d", q.MaxParams))
|
|
}
|
|
path := "/api/models?" + v.Encode()
|
|
|
|
c.mu.Lock()
|
|
if e, ok := c.cache[path]; ok && time.Since(e.at) < exploreTTL {
|
|
c.mu.Unlock()
|
|
return e.models, nil
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
var out []Model
|
|
if err := c.get(ctx, path, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
c.mu.Lock()
|
|
if c.cache == nil || len(c.cache) > 200 {
|
|
c.cache = map[string]cached{}
|
|
}
|
|
c.cache[path] = cached{at: time.Now(), models: out}
|
|
c.mu.Unlock()
|
|
return out, nil
|
|
}
|
|
|
|
// exploreTTL is how long a listing is reused.
|
|
const exploreTTL = 10 * time.Minute
|
|
|
|
type cached struct {
|
|
at time.Time
|
|
models []Model
|
|
}
|
|
|
|
// File is one file in a repository.
|
|
type File struct {
|
|
Path string
|
|
// Bytes is the real size. For a file stored in LFS — every weights file —
|
|
// the plain size field is the size of the pointer, a few hundred bytes, so
|
|
// reading that one would report a 5 GB model as 133 bytes.
|
|
Bytes int64
|
|
// SHA256 is what the registry publishes for an LFS file, so it can be
|
|
// compared with what the mirror computes after fetching. Empty for a small
|
|
// file stored in git, where the oid is a git blob hash and not a checksum
|
|
// of the content.
|
|
SHA256 string
|
|
}
|
|
|
|
// IsWeights reports whether this file is a model rather than documentation.
|
|
//
|
|
// By extension, which is crude and right: the registry does not label them, and
|
|
// an operator filling a mirror wants the four files out of thirty that are
|
|
// worth gigabytes.
|
|
func (f File) IsWeights() bool {
|
|
for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} {
|
|
if strings.HasSuffix(strings.ToLower(f.Path), ext) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Repo is a repository's current revision and what is in it.
|
|
type Repo struct {
|
|
ID string
|
|
// Revision is the commit the listing came from.
|
|
//
|
|
// Carried so a mirrored copy can name it. Fetching from "main" records
|
|
// nothing about what was fetched, and a repository that is re-uploaded
|
|
// under the same branch leaves two different sets of weights with the same
|
|
// provenance and no way to tell them apart.
|
|
Revision string
|
|
Gated Gated
|
|
Files []File
|
|
// Architecture is the GGUF architecture the registry read from the
|
|
// repository's weights, empty for a repository with none. What decides
|
|
// whether llama.cpp can load it, known before anything is downloaded.
|
|
Architecture string
|
|
}
|
|
|
|
// ShortRevision is the revision as a person would write it.
|
|
func (r Repo) ShortRevision() string {
|
|
if len(r.Revision) > 8 {
|
|
return r.Revision[:8]
|
|
}
|
|
return r.Revision
|
|
}
|
|
|
|
// Repo reads a repository's revision and file list.
|
|
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
|
id = strings.Trim(strings.TrimSpace(id), "/")
|
|
if !strings.Contains(id, "/") {
|
|
return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id)
|
|
}
|
|
|
|
var meta struct {
|
|
ID string `json:"id"`
|
|
SHA string `json:"sha"`
|
|
Gated Gated `json:"gated"`
|
|
GGUF *struct {
|
|
Architecture string `json:"architecture"`
|
|
} `json:"gguf"`
|
|
}
|
|
// Expanded to what this reads: asking for gguf alone would leave out the
|
|
// revision and the gate.
|
|
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
|
|
return Repo{}, err
|
|
}
|
|
|
|
var tree []struct {
|
|
Type string `json:"type"`
|
|
Path string `json:"path"`
|
|
Size int64 `json:"size"`
|
|
LFS *struct {
|
|
OID string `json:"oid"`
|
|
Size int64 `json:"size"`
|
|
} `json:"lfs"`
|
|
}
|
|
// By revision rather than by branch, so the listing and the URLs built from
|
|
// it describe the same commit even if the branch moves in between.
|
|
rev := meta.SHA
|
|
if rev == "" {
|
|
rev = "main"
|
|
}
|
|
if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil {
|
|
return Repo{}, err
|
|
}
|
|
|
|
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
|
|
if meta.GGUF != nil {
|
|
r.Architecture = meta.GGUF.Architecture
|
|
}
|
|
if r.ID == "" {
|
|
r.ID = id
|
|
}
|
|
for _, e := range tree {
|
|
if e.Type != "file" {
|
|
continue
|
|
}
|
|
f := File{Path: e.Path, Bytes: e.Size}
|
|
if e.LFS != nil {
|
|
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
|
|
}
|
|
r.Files = append(r.Files, f)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// DownloadURL is where the mirror should fetch a file from.
|
|
//
|
|
// Pinned to the revision the listing came from. "main" would mean the mirror
|
|
// fetches whatever is there when it gets round to it, which is not necessarily
|
|
// what was on screen when somebody pressed the button.
|
|
func (r Repo) DownloadURL(base, path string) string {
|
|
if base == "" {
|
|
base = DefaultEndpoint
|
|
}
|
|
rev := r.Revision
|
|
if rev == "" {
|
|
rev = "main"
|
|
}
|
|
return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path
|
|
}
|
|
|
|
func (c *Client) get(ctx context.Context, path string, out any) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := c.hc.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("reaching the model registry: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Capped: this is a third party, and a page that renders what it sends
|
|
// should not be handed a gigabyte of it.
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch {
|
|
case resp.StatusCode == http.StatusNotFound:
|
|
return fmt.Errorf("the registry has nothing at %s", path)
|
|
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
|
|
// Said plainly, because the next thing somebody does is check their
|
|
// network. It is the repository, not the connection.
|
|
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
|
|
case resp.StatusCode >= 300:
|
|
return fmt.Errorf("the registry answered %d", resp.StatusCode)
|
|
}
|
|
if err := json.Unmarshal(body, out); err != nil {
|
|
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// shardPattern matches the "-00002-of-00004" a split file carries before its
|
|
// extension. Five digits in practice, but not required: the count is what
|
|
// matters and a repository that used three would otherwise list every part as
|
|
// a separate model.
|
|
var shardPattern = regexp.MustCompile(`^(.*)-(\d+)-of-(\d+)(\.[^.]+)$`)
|
|
|
|
// FileSet is one thing worth fetching: a file, or every shard of one.
|
|
//
|
|
// The distinction the page turns on. Two GGUF files in a repository are usually
|
|
// alternatives — q4_k_m or q8_0, you want one — while two safetensors files are
|
|
// halves of the same weights and useless apart. Fetching "all the weights" is
|
|
// right for the second and wasteful for the first, so neither is offered:
|
|
// a set is offered, and a set is whole.
|
|
type FileSet struct {
|
|
// Name is what to call it. For shards it keeps the -of- notation, because
|
|
// a repository can hold both the split and the joined copy of the same
|
|
// quantisation and they must not read as one entry.
|
|
Name string
|
|
Files []File
|
|
Bytes int64
|
|
Shards int
|
|
}
|
|
|
|
// SHA256 is the checksum when there is one to give.
|
|
//
|
|
// Empty for a shard set: several files have several checksums and no single
|
|
// value for the set, and showing one of them invites comparing it against the
|
|
// wrong thing.
|
|
func (s FileSet) SHA256() string {
|
|
if len(s.Files) == 1 {
|
|
return s.Files[0].SHA256
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Group collects shards into sets, leaving every other file on its own.
|
|
//
|
|
// Order is preserved from the listing, so a repository that sorts its files
|
|
// sensibly still reads sensibly here.
|
|
func Group(files []File) []FileSet {
|
|
var out []FileSet
|
|
at := map[string]int{}
|
|
for _, f := range files {
|
|
m := shardPattern.FindStringSubmatch(f.Path)
|
|
if m == nil {
|
|
out = append(out, FileSet{Name: f.Path, Files: []File{f}, Bytes: f.Bytes})
|
|
continue
|
|
}
|
|
prefix, total, ext := m[1], m[3], m[4]
|
|
name := prefix + "-*-of-" + total + ext
|
|
i, ok := at[name]
|
|
if !ok {
|
|
at[name] = len(out)
|
|
out = append(out, FileSet{Name: name})
|
|
i = at[name]
|
|
}
|
|
out[i].Files = append(out[i].Files, f)
|
|
out[i].Bytes += f.Bytes
|
|
out[i].Shards = len(out[i].Files)
|
|
}
|
|
return out
|
|
}
|