Show a Hugging Face repository's model card on the mirror page
Opening a repository shows its model card, rendered from markdown, with its base model, license and languages above it; the listing names each repository's base model and license under its name. The card is a stranger's text: raw HTML and dangerous links are dropped, images are shown as links rather than loaded, and relative links point into the repository. Adds github.com/yuin/goldmark for the markdown. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
02fa639657
commit
3fea39feb9
10 files changed
+375
-27
No files matched your search
+135
-25
@@ -267,6 +267,16 @@ type Repo struct {
|
||||
// repository's weights, empty for a repository with none. What decides
|
||||
// whether llama.cpp can load it, known before anything is downloaded.
|
||||
Architecture string
|
||||
|
||||
// From the model card's header: what somebody asks before reading the
|
||||
// rest of it.
|
||||
License string
|
||||
BaseModel []string
|
||||
Languages []string
|
||||
// Card is the model card's text, markdown, without its header. Empty when
|
||||
// the repository has none or it could not be read — a repository is
|
||||
// still worth fetching from without its README.
|
||||
Card string
|
||||
}
|
||||
|
||||
// ShortRevision is the revision as a person would write it.
|
||||
@@ -291,10 +301,15 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
GGUF *struct {
|
||||
Architecture string `json:"architecture"`
|
||||
} `json:"gguf"`
|
||||
CardData struct {
|
||||
License string `json:"license"`
|
||||
BaseModel stringish `json:"base_model"`
|
||||
Language stringish `json:"language"`
|
||||
} `json:"cardData"`
|
||||
}
|
||||
// Expanded to what this reads: asking for gguf alone would leave out the
|
||||
// revision and the gate.
|
||||
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
|
||||
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil {
|
||||
return Repo{}, err
|
||||
}
|
||||
|
||||
@@ -317,26 +332,112 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
return Repo{}, err
|
||||
}
|
||||
|
||||
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
|
||||
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated,
|
||||
License: meta.CardData.License, BaseModel: meta.CardData.BaseModel,
|
||||
Languages: meta.CardData.Language}
|
||||
if meta.GGUF != nil {
|
||||
r.Architecture = meta.GGUF.Architecture
|
||||
}
|
||||
if r.ID == "" {
|
||||
r.ID = id
|
||||
}
|
||||
hasCard := false
|
||||
for _, e := range tree {
|
||||
if e.Type != "file" {
|
||||
continue
|
||||
}
|
||||
if e.Path == "README.md" {
|
||||
hasCard = true
|
||||
}
|
||||
f := File{Path: e.Path, Bytes: e.Size}
|
||||
if e.LFS != nil {
|
||||
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
|
||||
}
|
||||
r.Files = append(r.Files, f)
|
||||
}
|
||||
if hasCard {
|
||||
// Best effort: the files are what the page is for.
|
||||
if b, err := c.raw(ctx, "/"+r.ID+"/raw/"+rev+"/README.md", maxCard); err == nil {
|
||||
r.Card = stripFrontMatter(string(b))
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// maxCard is as much of a model card as is read. Most are a few kilobytes;
|
||||
// the cap is for the one that is not.
|
||||
const maxCard = 256 << 10
|
||||
|
||||
// stripFrontMatter drops a card's YAML header, which the registry already
|
||||
// hands over as cardData and which reads as noise at the top of the text.
|
||||
func stripFrontMatter(s string) string {
|
||||
s = strings.TrimPrefix(s, "\ufeff")
|
||||
if !strings.HasPrefix(s, "---") {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
rest := s[3:]
|
||||
if i := strings.Index(rest, "\n---"); i >= 0 {
|
||||
rest = rest[i+4:]
|
||||
if j := strings.IndexByte(rest, '\n'); j >= 0 {
|
||||
rest = rest[j+1:]
|
||||
} else {
|
||||
rest = ""
|
||||
}
|
||||
return strings.TrimSpace(rest)
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// stringish is a card field written as one string or a list of them.
|
||||
type stringish []string
|
||||
|
||||
// UnmarshalJSON accepts a string, a list of strings, or anything else as
|
||||
// nothing: a card header is free-form YAML, and one odd field should not
|
||||
// cost the page its file list.
|
||||
func (v *stringish) UnmarshalJSON(b []byte) error {
|
||||
var one string
|
||||
if json.Unmarshal(b, &one) == nil {
|
||||
if one != "" {
|
||||
*v = []string{one}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var many []string
|
||||
if json.Unmarshal(b, &many) == nil {
|
||||
*v = many
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BaseModel is the model a repository was made from, read from its tags:
|
||||
// "base_model:quantized:Qwen/Qwen3-8B" and its kin. Empty when not said.
|
||||
func (m Model) BaseModel() string {
|
||||
for _, t := range m.Tags {
|
||||
if !strings.HasPrefix(t, "base_model:") {
|
||||
continue
|
||||
}
|
||||
rest := strings.TrimPrefix(t, "base_model:")
|
||||
// The relation is optional: base_model:X or base_model:quantized:X.
|
||||
if i := strings.Index(rest, ":"); i >= 0 && !strings.Contains(rest[:i], "/") {
|
||||
rest = rest[i+1:]
|
||||
}
|
||||
if rest != "" {
|
||||
return rest
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// License is the license tag, empty when there is none.
|
||||
func (m Model) License() string {
|
||||
for _, t := range m.Tags {
|
||||
if strings.HasPrefix(t, "license:") {
|
||||
return strings.TrimPrefix(t, "license:")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// DownloadURL is where the mirror should fetch a file from.
|
||||
//
|
||||
// Pinned to the revision the listing came from. "main" would mean the mirror
|
||||
@@ -354,38 +455,47 @@ func (r Repo) DownloadURL(base, path string) string {
|
||||
}
|
||||
|
||||
func (c *Client) get(ctx context.Context, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
||||
body, err := c.raw(ctx, path, 4<<20)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reaching the model registry: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Capped: this is a third party, and a page that renders what it sends
|
||||
// should not be handed a gigabyte of it.
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusNotFound:
|
||||
return fmt.Errorf("the registry has nothing at %s", path)
|
||||
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
|
||||
// Said plainly, because the next thing somebody does is check their
|
||||
// network. It is the repository, not the connection.
|
||||
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
|
||||
case resp.StatusCode >= 300:
|
||||
return fmt.Errorf("the registry answered %d", resp.StatusCode)
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// raw reads a path from the registry, at most limit bytes of it.
|
||||
func (c *Client) raw(ctx context.Context, path string, limit int64) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reaching the model registry: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Capped: this is a third party, and a page that renders what it sends
|
||||
// should not be handed a gigabyte of it.
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusNotFound:
|
||||
return nil, fmt.Errorf("the registry has nothing at %s", path)
|
||||
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
|
||||
// Said plainly, because the next thing somebody does is check their
|
||||
// network. It is the repository, not the connection.
|
||||
return nil, fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
|
||||
case resp.StatusCode >= 300:
|
||||
return nil, fmt.Errorf("the registry answered %d", resp.StatusCode)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// shardPattern matches the "-00002-of-00004" a split file carries before its
|
||||
// extension. Five digits in practice, but not required: the count is what
|
||||
// matters and a repository that used three would otherwise list every part as
|
||||
|
||||
@@ -230,3 +230,39 @@ func TestExploringAsksForWhatThePageFilters(t *testing.T) {
|
||||
t.Errorf("the registry was asked %d times", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryCarriesItsCardWithoutTheHeader(t *testing.T) {
|
||||
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasPrefix(r.URL.Path, "/api/models/q/m/tree/"):
|
||||
_, _ = w.Write([]byte(`[{"type":"file","path":"README.md","size":40},{"type":"file","path":"m.gguf","size":1}]`))
|
||||
case r.URL.Path == "/api/models/q/m":
|
||||
_, _ = w.Write([]byte(`{"id":"q/m","sha":"abc","cardData":{"license":"apache-2.0","base_model":"Qwen/Qwen2.5-0.5B","language":["en","tr"]}}`))
|
||||
case r.URL.Path == "/q/m/raw/abc/README.md":
|
||||
_, _ = w.Write([]byte("---\nlicense: apache-2.0\n---\n\n# M\n\nA model.\n"))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
got, err := c.Repo(t.Context(), "q/m")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Card != "# M\n\nA model." {
|
||||
t.Errorf("card = %q", got.Card)
|
||||
}
|
||||
if got.License != "apache-2.0" || len(got.BaseModel) != 1 || got.BaseModel[0] != "Qwen/Qwen2.5-0.5B" ||
|
||||
len(got.Languages) != 2 {
|
||||
t.Errorf("card header = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAListingNamesTheBaseModelAndLicenseFromItsTags(t *testing.T) {
|
||||
m := Model{Tags: []string{"gguf", "base_model:quantized:Qwen/Qwen3-8B", "base_model:Qwen/Qwen3-8B", "license:apache-2.0"}}
|
||||
if m.BaseModel() != "Qwen/Qwen3-8B" || m.License() != "apache-2.0" {
|
||||
t.Errorf("base %q, license %q", m.BaseModel(), m.License())
|
||||
}
|
||||
if (Model{Tags: []string{"base_model:org/x"}}).BaseModel() != "org/x" {
|
||||
t.Error("a base model with no relation should read as itself")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user