Show a Hugging Face repository's model card on the mirror page

Opening a repository shows its model card, rendered from markdown, with
its base model, license and languages above it; the listing names each
repository's base model and license under its name. The card is a
stranger's text: raw HTML and dangerous links are dropped, images are
shown as links rather than loaded, and relative links point into the
repository.

Adds github.com/yuin/goldmark for the markdown.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 22:03:19 +02:00
1 parent 02fa639657
commit 3fea39feb9
10 files changed
+375 -27

No files matched your search

+135 -25
View File
@@ -267,6 +267,16 @@ type Repo struct {
// repository's weights, empty for a repository with none. What decides
// whether llama.cpp can load it, known before anything is downloaded.
Architecture string
// From the model card's header: what somebody asks before reading the
// rest of it.
License string
BaseModel []string
Languages []string
// Card is the model card's text, markdown, without its header. Empty when
// the repository has none or it could not be read — a repository is
// still worth fetching from without its README.
Card string
}
// ShortRevision is the revision as a person would write it.
@@ -291,10 +301,15 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
GGUF *struct {
Architecture string `json:"architecture"`
} `json:"gguf"`
CardData struct {
License string `json:"license"`
BaseModel stringish `json:"base_model"`
Language stringish `json:"language"`
} `json:"cardData"`
}
// Expanded to what this reads: asking for gguf alone would leave out the
// revision and the gate.
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf", &meta); err != nil {
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil {
return Repo{}, err
}
@@ -317,26 +332,112 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
return Repo{}, err
}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated}
r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated,
License: meta.CardData.License, BaseModel: meta.CardData.BaseModel,
Languages: meta.CardData.Language}
if meta.GGUF != nil {
r.Architecture = meta.GGUF.Architecture
}
if r.ID == "" {
r.ID = id
}
hasCard := false
for _, e := range tree {
if e.Type != "file" {
continue
}
if e.Path == "README.md" {
hasCard = true
}
f := File{Path: e.Path, Bytes: e.Size}
if e.LFS != nil {
f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID
}
r.Files = append(r.Files, f)
}
if hasCard {
// Best effort: the files are what the page is for.
if b, err := c.raw(ctx, "/"+r.ID+"/raw/"+rev+"/README.md", maxCard); err == nil {
r.Card = stripFrontMatter(string(b))
}
}
return r, nil
}
// maxCard is as much of a model card as is read. Most are a few kilobytes;
// the cap is for the one that is not.
const maxCard = 256 << 10
// stripFrontMatter drops a card's YAML header, which the registry already
// hands over as cardData and which reads as noise at the top of the text.
func stripFrontMatter(s string) string {
s = strings.TrimPrefix(s, "\ufeff")
if !strings.HasPrefix(s, "---") {
return strings.TrimSpace(s)
}
rest := s[3:]
if i := strings.Index(rest, "\n---"); i >= 0 {
rest = rest[i+4:]
if j := strings.IndexByte(rest, '\n'); j >= 0 {
rest = rest[j+1:]
} else {
rest = ""
}
return strings.TrimSpace(rest)
}
return strings.TrimSpace(s)
}
// stringish is a card field written as one string or a list of them.
type stringish []string
// UnmarshalJSON accepts a string, a list of strings, or anything else as
// nothing: a card header is free-form YAML, and one odd field should not
// cost the page its file list.
func (v *stringish) UnmarshalJSON(b []byte) error {
var one string
if json.Unmarshal(b, &one) == nil {
if one != "" {
*v = []string{one}
}
return nil
}
var many []string
if json.Unmarshal(b, &many) == nil {
*v = many
}
return nil
}
// BaseModel is the model a repository was made from, read from its tags:
// "base_model:quantized:Qwen/Qwen3-8B" and its kin. Empty when not said.
func (m Model) BaseModel() string {
for _, t := range m.Tags {
if !strings.HasPrefix(t, "base_model:") {
continue
}
rest := strings.TrimPrefix(t, "base_model:")
// The relation is optional: base_model:X or base_model:quantized:X.
if i := strings.Index(rest, ":"); i >= 0 && !strings.Contains(rest[:i], "/") {
rest = rest[i+1:]
}
if rest != "" {
return rest
}
}
return ""
}
// License is the license tag, empty when there is none.
func (m Model) License() string {
for _, t := range m.Tags {
if strings.HasPrefix(t, "license:") {
return strings.TrimPrefix(t, "license:")
}
}
return ""
}
// DownloadURL is where the mirror should fetch a file from.
//
// Pinned to the revision the listing came from. "main" would mean the mirror
@@ -354,38 +455,47 @@ func (r Repo) DownloadURL(base, path string) string {
}
func (c *Client) get(ctx context.Context, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
body, err := c.raw(ctx, path, 4<<20)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return fmt.Errorf("reaching the model registry: %w", err)
}
defer resp.Body.Close()
// Capped: this is a third party, and a page that renders what it sends
// should not be handed a gigabyte of it.
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return err
}
switch {
case resp.StatusCode == http.StatusNotFound:
return fmt.Errorf("the registry has nothing at %s", path)
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
// Said plainly, because the next thing somebody does is check their
// network. It is the repository, not the connection.
return fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
case resp.StatusCode >= 300:
return fmt.Errorf("the registry answered %d", resp.StatusCode)
}
if err := json.Unmarshal(body, out); err != nil {
return fmt.Errorf("the registry's answer was not the JSON expected: %w", err)
}
return nil
}
// raw reads a path from the registry, at most limit bytes of it.
func (c *Client) raw(ctx context.Context, path string, limit int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
if err != nil {
return nil, err
}
resp, err := c.hc.Do(req)
if err != nil {
return nil, fmt.Errorf("reaching the model registry: %w", err)
}
defer resp.Body.Close()
// Capped: this is a third party, and a page that renders what it sends
// should not be handed a gigabyte of it.
body, err := io.ReadAll(io.LimitReader(resp.Body, limit))
if err != nil {
return nil, err
}
switch {
case resp.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("the registry has nothing at %s", path)
case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:
// Said plainly, because the next thing somebody does is check their
// network. It is the repository, not the connection.
return nil, fmt.Errorf("the registry refused this anonymously; the repository is gated or private")
case resp.StatusCode >= 300:
return nil, fmt.Errorf("the registry answered %d", resp.StatusCode)
}
return body, nil
}
// shardPattern matches the "-00002-of-00004" a split file carries before its
// extension. Five digits in practice, but not required: the count is what
// matters and a repository that used three would otherwise list every part as
+36
View File
@@ -230,3 +230,39 @@ func TestExploringAsksForWhatThePageFilters(t *testing.T) {
t.Errorf("the registry was asked %d times", calls)
}
}
func TestARepositoryCarriesItsCardWithoutTheHeader(t *testing.T) {
c := registry(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/api/models/q/m/tree/"):
_, _ = w.Write([]byte(`[{"type":"file","path":"README.md","size":40},{"type":"file","path":"m.gguf","size":1}]`))
case r.URL.Path == "/api/models/q/m":
_, _ = w.Write([]byte(`{"id":"q/m","sha":"abc","cardData":{"license":"apache-2.0","base_model":"Qwen/Qwen2.5-0.5B","language":["en","tr"]}}`))
case r.URL.Path == "/q/m/raw/abc/README.md":
_, _ = w.Write([]byte("---\nlicense: apache-2.0\n---\n\n# M\n\nA model.\n"))
default:
http.NotFound(w, r)
}
})
got, err := c.Repo(t.Context(), "q/m")
if err != nil {
t.Fatal(err)
}
if got.Card != "# M\n\nA model." {
t.Errorf("card = %q", got.Card)
}
if got.License != "apache-2.0" || len(got.BaseModel) != 1 || got.BaseModel[0] != "Qwen/Qwen2.5-0.5B" ||
len(got.Languages) != 2 {
t.Errorf("card header = %+v", got)
}
}
func TestAListingNamesTheBaseModelAndLicenseFromItsTags(t *testing.T) {
m := Model{Tags: []string{"gguf", "base_model:quantized:Qwen/Qwen3-8B", "base_model:Qwen/Qwen3-8B", "license:apache-2.0"}}
if m.BaseModel() != "Qwen/Qwen3-8B" || m.License() != "apache-2.0" {
t.Errorf("base %q, license %q", m.BaseModel(), m.License())
}
if (Model{Tags: []string{"base_model:org/x"}}).BaseModel() != "org/x" {
t.Error("a base model with no relation should read as itself")
}
}