diff --git a/docs/reference.md b/docs/reference.md index 183b052..06dfb00 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -80,6 +80,7 @@ rather than repeated. | `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) | | `api` | `string` | [Mirror](#mirror) | | `tls` | `tlsconf.Config` | [Mirror](#mirror) | +| `registry` | `string` | [Mirror](#mirror) | | `timeout` | `time.Duration` | [Mirror](#mirror) | | `name` | `string` | [Model](#model) | | `aliases` | `[]string` | [Model](#model) | @@ -603,6 +604,16 @@ the mirror says as much in its own log when it starts that way. Takes a [Config](#config) block. +### `registry` + +`string` + +Registry is the model registry the mirror page browses, empty for the +public Hugging Face. Set it to an internal mirror of the registry where +the control plane cannot reach the internet — or leave the whole mirror +block out, which turns the explorer off rather than giving it a search +box that can only time out. + ### `timeout` `time.Duration` diff --git a/internal/cli/server.go b/internal/cli/server.go index 37ae0af..7932557 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -27,6 +27,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/router" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -197,6 +198,10 @@ func newServerCmd() *cobra.Command { return fmt.Errorf("the model mirror: %w", err) } rt.SetMirror(mc) + // Browsing the registry is what turns "paste a resolve URL" + // into "find the model". It needs a way out to the internet + // from this machine, which the mirror's own fetching does not. + rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)) log.Info("model mirror configured", "api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled()) } diff --git a/internal/config/config.go b/internal/config/config.go index 2292f1e..80da07f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -99,6 +99,12 @@ type Mirror struct { // acceptable on a network where nothing untrusted can reach that port — // the mirror says as much in its own log when it starts that way. TLS tlsconf.Config `koanf:"tls"` + // Registry is the model registry the mirror page browses, empty for the + // public Hugging Face. Set it to an internal mirror of the registry where + // the control plane cannot reach the internet — or leave the whole mirror + // block out, which turns the explorer off rather than giving it a search + // box that can only time out. + Registry string `koanf:"registry"` // Timeout bounds a management call. It does not bound a download: the // mirror answers a fetch as soon as it has started one, and the bytes keep // arriving long after this deadline has passed. diff --git a/internal/hf/hf.go b/internal/hf/hf.go new file mode 100644 index 0000000..775fac2 --- /dev/null +++ b/internal/hf/hf.go @@ -0,0 +1,259 @@ +// Package hf reads the Hugging Face model registry. +// +// Enough of it to answer two questions an operator has before mirroring a +// model: what is there, and which file do I want. Nothing here downloads +// anything — the mirror does that, over its own network, and the point of the +// split is that the control plane never carries a model's bytes. +// +// Read-only and anonymous. A gated or private repository answers 401 to this +// client, which is reported rather than worked around: a token belongs to the +// machine that fetches the weights, not to the one browsing a catalogue. +package hf + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// DefaultEndpoint is the public registry. +const DefaultEndpoint = "https://huggingface.co" + +// Client reads the registry. +type Client struct { + base string + hc *http.Client +} + +// New builds a client against an endpoint, or the public one when empty. +func New(endpoint string, timeout time.Duration) *Client { + if endpoint == "" { + endpoint = DefaultEndpoint + } + if timeout <= 0 { + // Short. This runs while somebody is looking at a page, and a registry + // that is slow today is better reported than waited for. + timeout = 10 * time.Second + } + return &Client{base: strings.TrimSuffix(endpoint, "/"), + hc: &http.Client{Timeout: timeout}} +} + +// Model is a repository as a search result shows it. +type Model struct { + ID string `json:"id"` + Downloads int64 `json:"downloads"` + Likes int64 `json:"likes"` + PipelineTag string `json:"pipeline_tag"` + Tags []string `json:"tags"` + LastModified time.Time `json:"lastModified"` + // Gated is true when the repository asks for terms to be accepted. It + // decodes from either a bool or a string ("auto", "manual"), which is why + // it is not a plain bool here. + Gated Gated `json:"gated"` + Private bool `json:"private"` +} + +// Gated is Hugging Face's gating flag, which is false or a word. +type Gated struct{ Reason string } + +// UnmarshalJSON accepts false, true, or a string. +// +// The field is documented as a boolean and is a string in practice for most +// gated repositories. Decoding it as a bool fails the whole response, which +// would make one gated model in a page of results hide the other nineteen. +func (g *Gated) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err == nil { + g.Reason = s + return nil + } + var v bool + if err := json.Unmarshal(b, &v); err != nil { + return nil // unknown shape: treat as not gated rather than fail the page + } + if v { + g.Reason = "yes" + } + return nil +} + +// Is reports whether the repository is gated at all. +func (g Gated) Is() bool { return g.Reason != "" && g.Reason != "false" } + +// Search finds repositories by name. +func (c *Client) Search(ctx context.Context, query string, limit int) ([]Model, error) { + if strings.TrimSpace(query) == "" { + return nil, fmt.Errorf("search for something") + } + if limit <= 0 || limit > 50 { + limit = 20 + } + q := url.Values{ + "search": {query}, + "limit": {fmt.Sprint(limit)}, + // By downloads, because the thing somebody means by a short name is + // almost always the one everybody else meant too. + "sort": {"downloads"}, + "direction": {"-1"}, + } + var out []Model + err := c.get(ctx, "/api/models?"+q.Encode(), &out) + return out, err +} + +// File is one file in a repository. +type File struct { + Path string + // Bytes is the real size. For a file stored in LFS — every weights file — + // the plain size field is the size of the pointer, a few hundred bytes, so + // reading that one would report a 5 GB model as 133 bytes. + Bytes int64 + // SHA256 is what the registry publishes for an LFS file, so it can be + // compared with what the mirror computes after fetching. Empty for a small + // file stored in git, where the oid is a git blob hash and not a checksum + // of the content. + SHA256 string +} + +// IsWeights reports whether this file is a model rather than documentation. +// +// By extension, which is crude and right: the registry does not label them, and +// an operator filling a mirror wants the four files out of thirty that are +// worth gigabytes. +func (f File) IsWeights() bool { + for _, ext := range []string{".gguf", ".safetensors", ".bin", ".pt", ".pth"} { + if strings.HasSuffix(strings.ToLower(f.Path), ext) { + return true + } + } + return false +} + +// Repo is a repository's current revision and what is in it. +type Repo struct { + ID string + // Revision is the commit the listing came from. + // + // Carried so a mirrored copy can name it. Fetching from "main" records + // nothing about what was fetched, and a repository that is re-uploaded + // under the same branch leaves two different sets of weights with the same + // provenance and no way to tell them apart. + Revision string + Gated Gated + Files []File +} + +// ShortRevision is the revision as a person would write it. +func (r Repo) ShortRevision() string { + if len(r.Revision) > 8 { + return r.Revision[:8] + } + return r.Revision +} + +// Repo reads a repository's revision and file list. +func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { + id = strings.Trim(strings.TrimSpace(id), "/") + if !strings.Contains(id, "/") { + return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id) + } + + var meta struct { + ID string `json:"id"` + SHA string `json:"sha"` + Gated Gated `json:"gated"` + } + if err := c.get(ctx, "/api/models/"+id, &meta); err != nil { + return Repo{}, err + } + + var tree []struct { + Type string `json:"type"` + Path string `json:"path"` + Size int64 `json:"size"` + LFS *struct { + OID string `json:"oid"` + Size int64 `json:"size"` + } `json:"lfs"` + } + // By revision rather than by branch, so the listing and the URLs built from + // it describe the same commit even if the branch moves in between. + rev := meta.SHA + if rev == "" { + rev = "main" + } + if err := c.get(ctx, "/api/models/"+id+"/tree/"+rev, &tree); err != nil { + return Repo{}, err + } + + r := Repo{ID: meta.ID, Revision: meta.SHA, Gated: meta.Gated} + if r.ID == "" { + r.ID = id + } + for _, e := range tree { + if e.Type != "file" { + continue + } + f := File{Path: e.Path, Bytes: e.Size} + if e.LFS != nil { + f.Bytes, f.SHA256 = e.LFS.Size, e.LFS.OID + } + r.Files = append(r.Files, f) + } + return r, nil +} + +// DownloadURL is where the mirror should fetch a file from. +// +// Pinned to the revision the listing came from. "main" would mean the mirror +// fetches whatever is there when it gets round to it, which is not necessarily +// what was on screen when somebody pressed the button. +func (r Repo) DownloadURL(base, path string) string { + if base == "" { + base = DefaultEndpoint + } + rev := r.Revision + if rev == "" { + rev = "main" + } + return strings.TrimSuffix(base, "/") + "/" + r.ID + "/resolve/" + rev + "/" + path +} + +func (c *Client) get(ctx context.Context, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil) + if err != nil { + return err + } + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("reaching the model registry: %w", err) + } + defer resp.Body.Close() + + // Capped: this is a third party, and a page that renders what it sends + // should not be handed a gigabyte of it. + body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if err != nil { + return err + } + switch { + case resp.StatusCode == http.StatusNotFound: + return fmt.Errorf("the registry has nothing at %s", path) + case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden: + // Said plainly, because the next thing somebody does is check their + // network. It is the repository, not the connection. + return fmt.Errorf("the registry refused this anonymously; the repository is gated or private") + case resp.StatusCode >= 300: + return fmt.Errorf("the registry answered %d", resp.StatusCode) + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("the registry's answer was not the JSON expected: %w", err) + } + return nil +} diff --git a/internal/hf/hf_test.go b/internal/hf/hf_test.go new file mode 100644 index 0000000..fd68845 --- /dev/null +++ b/internal/hf/hf_test.go @@ -0,0 +1,138 @@ +package hf + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// registry stands in for Hugging Face, answering the two endpoints used. +func registry(t *testing.T, h http.HandlerFunc) *Client { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + return New(srv.URL, time.Second) +} + +func TestTheRealSizeOfAWeightsFileIsTheLFSOne(t *testing.T) { + c := registry(t, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/tree/abc123") { + _, _ = w.Write([]byte(`[ + {"type":"file","path":"README.md","size":900,"oid":"gitblob"}, + {"type":"file","path":"model.gguf","size":133, + "lfs":{"oid":"deadbeef","size":1117320736}}]`)) + return + } + _, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`)) + }) + + repo, err := c.Repo(t.Context(), "Qwen/X") + if err != nil { + t.Fatal(err) + } + var gguf File + for _, f := range repo.Files { + if f.Path == "model.gguf" { + gguf = f + } + } + // The plain size of an LFS file is the size of the pointer. Reading that + // one reports a gigabyte model as 133 bytes, which is the number somebody + // would then plan disk against. + if gguf.Bytes != 1117320736 { + t.Errorf("expected the LFS size, got %d", gguf.Bytes) + } + if gguf.SHA256 != "deadbeef" { + t.Errorf("expected the published checksum, got %q", gguf.SHA256) + } +} + +func TestASmallFileHasNoChecksum(t *testing.T) { + c := registry(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/tree/") { + _, _ = w.Write([]byte(`[{"type":"file","path":"README.md","size":900,"oid":"gitblob"}]`)) + return + } + _, _ = w.Write([]byte(`{"id":"Qwen/X","sha":"abc123"}`)) + }) + + repo, _ := c.Repo(t.Context(), "Qwen/X") + // A git blob hash is not a checksum of the content, so reporting it as one + // would invite somebody to compare it with what the mirror computes and + // conclude the download was corrupt. + if repo.Files[0].SHA256 != "" { + t.Errorf("a git-stored file should publish no checksum, got %q", repo.Files[0].SHA256) + } +} + +func TestTheDownloadURLIsPinnedToTheRevisionThatWasListed(t *testing.T) { + r := Repo{ID: "Qwen/X", Revision: "abc123def456"} + got := r.DownloadURL("https://hf.example", "model.gguf") + + // "main" would mean the mirror fetches whatever is there when it gets round + // to it, which is not necessarily what was on screen when the button was + // pressed. + if !strings.Contains(got, "/resolve/abc123def456/") { + t.Errorf("the URL should pin the revision; got %s", got) + } +} + +func TestAGatedRepositorySaysSoRatherThanBlamingTheNetwork(t *testing.T) { + c := registry(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + + _, err := c.Repo(t.Context(), "meta/gated") + if err == nil || !strings.Contains(err.Error(), "gated or private") { + t.Errorf("expected the reason to be named; got %v", err) + } +} + +func TestOneGatedResultDoesNotHideThePage(t *testing.T) { + c := registry(t, func(w http.ResponseWriter, _ *http.Request) { + // The field is documented as a boolean and is a word in practice. + // Decoding it strictly fails the whole response, so one gated model in + // a page of twenty would hide the other nineteen. + _, _ = w.Write([]byte(`[{"id":"a","gated":false},{"id":"b","gated":"manual"}]`)) + }) + + got, err := c.Search(t.Context(), "qwen", 20) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("expected both results, got %d", len(got)) + } + if got[0].Gated.Is() || !got[1].Gated.Is() { + t.Errorf("gating did not decode: %+v", got) + } +} + +func TestSomethingThatIsNotARepositoryIsRefusedBeforeTheCall(t *testing.T) { + called := false + c := registry(t, func(http.ResponseWriter, *http.Request) { called = true }) + + if _, err := c.Repo(t.Context(), "qwen"); err == nil { + t.Error("a bare name is not owner/name and should be refused") + } + if called { + t.Error("nothing should have been requested") + } +} + +func TestWeightsAreToldApartFromDocumentation(t *testing.T) { + // Crude and right: the registry does not label them, and an operator + // filling a mirror wants the four files out of thirty worth gigabytes. + for _, f := range []string{"model.gguf", "model-00001-of-00002.safetensors", "pytorch_model.bin"} { + if !(File{Path: f}).IsWeights() { + t.Errorf("%s should count as weights", f) + } + } + for _, f := range []string{"README.md", "config.json", ".gitattributes", "tokenizer.model"} { + if (File{Path: f}).IsWeights() { + t.Errorf("%s should not count as weights", f) + } + } +} diff --git a/internal/mirror/api_test.go b/internal/mirror/api_test.go index 0aceda9..4252b04 100644 --- a/internal/mirror/api_test.go +++ b/internal/mirror/api_test.go @@ -326,3 +326,111 @@ func TestTheChecksumSurvivesTheDownloadFinishing(t *testing.T) { t.Errorf("the checksum should still be reported once the file is present; got %v", got) } } + +func TestTwoShardsOfOneVersionDownloadTogether(t *testing.T) { + a, _ := testAPI(t) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-release + _, _ = w.Write([]byte("weights")) + })) + defer srv.Close() + defer close(release) + + for _, shard := range []string{"part-1.safetensors", "part-2.safetensors"} { + code, body := call(t, a, "POST", "/v1/models", `{ + "source": "`+srv.URL+`/`+shard+`", "model": "llama", "version": "v1", + "file": "`+shard+`"}`) + // Keyed by version, the second shard was refused as already fetching + // until the first had finished — so a fifteen-shard model had to be + // pulled one shard at a time, each waiting on the last. + if code != http.StatusAccepted { + t.Fatalf("%s was refused with %d: %v", shard, code, body) + } + st := body["status"].(map[string]any) + if st["file"] != shard { + t.Errorf("expected %s to be its own download, got %v", shard, st["file"]) + } + } +} + +func TestAskingTwiceForTheSameFileStillJoinsTheOneRunning(t *testing.T) { + a, _ := testAPI(t) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-release + _, _ = w.Write([]byte("weights")) + })) + defer srv.Close() + defer close(release) + + body := `{"source": "` + srv.URL + `/f.gguf", "model": "m", "version": "v", "file": "f.gguf"}` + call(t, a, "POST", "/v1/models", body) + waitFor(t, "the first to register", func() bool { + st, ok := a.m.Status("m", "v") + return ok && st.State == Fetching + }) + + // Per file is still single-flight: two writers on one .part file is the + // thing the key exists to prevent. + code, second := call(t, a, "POST", "/v1/models", body) + if code != http.StatusAccepted { + t.Fatalf("expected the running download to be reported, got %d", code) + } + if st := second["status"].(map[string]any); st["state"] != string(Fetching) { + t.Errorf("expected the one already running, got %v", st["state"]) + } +} + +func TestAVersionIsOnlyPresentOnceEveryShardIs(t *testing.T) { + a, root := testAPI(t) + writeFile(t, filepath.Join(root, "llama", "v1", "part-1.safetensors"), "a") + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-release + _, _ = w.Write([]byte("b")) + })) + defer srv.Close() + defer close(release) + + call(t, a, "POST", "/v1/models", `{"source": "`+srv.URL+`/part-2.safetensors", + "model": "llama", "version": "v1", "file": "part-2.safetensors"}`) + waitFor(t, "the second shard to register", func() bool { + st, _ := a.m.Status("llama", "v1") + return st.State == Fetching + }) + + // One shard on disk and one still arriving is not a version anybody can + // load, so the version reports the worse of the two. + _, body := call(t, a, "GET", "/v1/models/llama/v1", "") + if st := body["status"].(map[string]any); st["state"] != string(Fetching) { + t.Errorf("expected the version to read as fetching, got %v", st["state"]) + } +} + +func TestRemovingAVersionIsRefusedWhileAnyShardIsArriving(t *testing.T) { + a, root := testAPI(t) + writeFile(t, filepath.Join(root, "llama", "v1", "part-1.safetensors"), "a") + + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + <-release + _, _ = w.Write([]byte("b")) + })) + defer srv.Close() + defer close(release) + + call(t, a, "POST", "/v1/models", `{"source": "`+srv.URL+`/part-2.safetensors", + "model": "llama", "version": "v1", "file": "part-2.safetensors"}`) + waitFor(t, "the shard to register", func() bool { + st, _ := a.m.Status("llama", "v1") + return st.State == Fetching + }) + + // Deleting the shard that already landed while its sibling is still coming + // leaves a directory that reappears half-populated. + if code, _ := call(t, a, "DELETE", "/v1/models/llama/v1", ""); code != http.StatusConflict { + t.Errorf("expected 409 while a shard is arriving, got %d", code) + } +} diff --git a/internal/mirror/manager.go b/internal/mirror/manager.go index 10ac5e7..ff4ee06 100644 --- a/internal/mirror/manager.go +++ b/internal/mirror/manager.go @@ -62,12 +62,21 @@ type Manager struct { log *slog.Logger mu sync.Mutex - // running is keyed by model/version, which is how a second request for a - // download already in flight joins it rather than starting a rival one + // running is keyed by model/version/file, which is how a second request for + // a download already in flight joins it rather than starting a rival one // writing to the same file. + // + // By file, not by version. A model is routinely several files — sharded + // safetensors, or two quantisations somebody wants side by side — and + // keyed by version the second shard was refused as "already fetching" + // until the first had finished. A fifteen-shard model had to be pulled one + // shard at a time, each waiting on the last. running map[string]*Status } +// runKey identifies one download. +func runKey(e Entry) string { return e.Model + "/" + e.Version + "/" + e.File } + // NewManager prepares a manager over a mirror root. func NewManager(root string, log *slog.Logger) *Manager { if log == nil { @@ -88,7 +97,7 @@ func (m *Manager) Start(source string, e Entry) (Status, error) { if source == "" { return Status{}, fmt.Errorf("a source URL is required") } - key := e.Model + "/" + e.Version + key := runKey(e) m.mu.Lock() if st, ok := m.running[key]; ok && st.State == Fetching { @@ -153,31 +162,83 @@ func (m *Manager) run(source string, e Entry, key string) { // The disk is the truth: a restart forgets every download, and a model that is // present should not be reported as unknown merely because this process did not // fetch it. +// A version of several files is reported as one: fetching while any file still +// is, failed if any failed, and the byte counts summed. The caller asked about +// a version, and a version with one shard still arriving is not present. func (m *Manager) Status(model, version string) (Status, bool) { - key := model + "/" + version + var out Status + found := false m.mu.Lock() - st, ok := m.running[key] - if ok { - out := *st - m.mu.Unlock() - return out, true + for _, st := range m.running { + if st.Model == model && st.Version == version { + out, found = merge(out, *st, found) + } } m.mu.Unlock() entries, err := List(m.root) if err != nil { - return Status{}, false + return out, found } for _, e := range entries { - if e.Model == model && e.Version == version { - return Status{ - Model: e.Model, Version: e.Version, File: e.File, - State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, - }, true + if e.Model != model || e.Version != version { + continue } + if m.isRunning(model, version, e.File) { + continue // already counted, and the live entry knows more + } + out, found = merge(out, Status{ + Model: e.Model, Version: e.Version, File: e.File, + State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, + }, found) + } + return out, found +} + +func (m *Manager) isRunning(model, version, file string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.running[model+"/"+version+"/"+file] + return ok +} + +// merge folds one file's status into a version's. +// +// The worst state wins: a version is only present once every file of it is, and +// a shard that failed makes the version unusable however many others arrived. +func merge(acc, one Status, have bool) (Status, bool) { + if !have { + return one, true + } + acc.BytesDone += one.BytesDone + acc.BytesTotal += one.BytesTotal + if worse(one.State) > worse(acc.State) { + acc.State, acc.Err, acc.File = one.State, one.Err, one.File + } + if acc.SHA256 != "" && one.SHA256 != "" { + // Two files, two checksums, and no single value for the version. Left + // empty rather than showing one of them, which would look like the + // version's and be compared against the wrong thing. + acc.SHA256 = "" + } else if one.SHA256 != "" { + acc.SHA256 = one.SHA256 + } + if one.FinishedAt.After(acc.FinishedAt) { + acc.FinishedAt = one.FinishedAt + } + return acc, true +} + +func worse(s State) int { + switch s { + case Failed: + return 2 + case Fetching: + return 1 + default: + return 0 } - return Status{}, false } // All reports everything the mirror holds or is fetching. @@ -202,7 +263,7 @@ func (m *Manager) All() ([]Status, error) { out := make([]Status, 0, len(entries)) seen := map[string]struct{}{} for _, e := range entries { - key := e.Model + "/" + e.Version + key := e.Model + "/" + e.Version + "/" + e.File st := Status{ Model: e.Model, Version: e.Version, File: e.File, State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, @@ -212,7 +273,7 @@ func (m *Manager) All() ([]Status, error) { // an operator compares against what upstream published. Dropping it in // favour of the disk entry lost it at the exact moment the file became // verifiable — the download finishing was what made it disappear. - if k, ok := known[key]; ok && k.File == e.File { + if k, ok := known[key]; ok { st.SHA256, st.Source = k.SHA256, k.Source } out = append(out, st) @@ -235,7 +296,14 @@ func (m *Manager) All() ([]Status, error) { func (m *Manager) Forget(model, version string) { m.mu.Lock() defer m.mu.Unlock() - delete(m.running, model+"/"+version) + // Every file of it. The version is the unit an operator retries, and + // leaving one shard's failure remembered would refuse the retry it was + // meant to allow. + for key, st := range m.running { + if st.Model == model && st.Version == version { + delete(m.running, key) + } + } } // RemoveVersion deletes every file under one model version and reports what diff --git a/internal/router/handlers.go b/internal/router/handlers.go index f0f3381..769ff3b 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -165,7 +165,7 @@ func (r *Router) Handler() http.Handler { // with its own disk, and it is worth administering whether or not this // process also has a cluster to show. if r.mirror != nil { - pages.RegisterAdminMirror(mux, r.mirror) + pages.RegisterAdminMirror(mux, r.mirror, r.registry) } } diff --git a/internal/router/router.go b/internal/router/router.go index 38ec0d1..1137a5a 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -102,6 +102,9 @@ type Router struct { // /admin/mirror unmounted — the right page for a deployment with no mirror // is no page, not an empty one pointing at nothing. mirror web.MirrorAdmin + // registry browses a model registry from the mirror page. Nil where the + // deployment has no way out to the internet. + registry web.Registry } // SetClusterView gives the router the data the /admin page renders. @@ -119,6 +122,9 @@ func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) { // cluster view: the router is built before the things its admin pages show. func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m } +// SetRegistry gives the mirror page somewhere to look models up. +func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } + // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } diff --git a/internal/web/account.go b/internal/web/account.go index b44d61f..2baf01c 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -57,6 +57,8 @@ type AccountPages struct { hasMirror bool // hasQuotas is set when /admin/quotas is registered. hasQuotas bool + // registry browses a model registry, or is nil where there is no way out. + registry Registry } // ServedModels supplies the names the profile page's example should use. diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index b642952..b4a6fea 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -27,6 +28,25 @@ type MirrorAdmin interface { Addr() string } +// Registry is the model registry the explorer browses. Nil leaves the explorer +// off, which is right for a deployment with no way out to the internet: a +// search box that can only time out is worse than no search box. +type Registry interface { + Search(ctx context.Context, query string, limit int) ([]hf.Model, error) + Repo(ctx context.Context, id string) (hf.Repo, error) +} + +// repoFile is one file of a browsed repository, ready to be mirrored. +type repoFile struct { + hf.File + Source string + // Model and Version are what the fetch form starts with. Editable, because + // the name a catalog entry uses is the operator's to choose and the + // registry's idea of it is only a good guess. + Model string + Version string +} + // mirrorRow is one model version as the page shows it. type mirrorRow struct { mirror.Status @@ -54,6 +74,14 @@ type adminMirrorData struct { // a mirror waiting to be filled, the other is a machine to go and look at. Unreachable bool + // The explorer. + HasRegistry bool + Query string + Results []hf.Model + Repo *hf.Repo + RepoFiles []repoFile + RegistryErr string + Message string Error string CSRF string @@ -73,8 +101,9 @@ type adminMirrorData struct { // need a shell for, while anything that would let a stolen session reach // further — pointing the control plane at a different mirror, say — stays in // configuration where changing it means access to the host. -func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin) { +func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, reg Registry) { p.hasMirror = true + p.registry = reg mux.HandleFunc("GET /admin/mirror", p.adminOnly( func(w http.ResponseWriter, r *http.Request, _ store.User) { @@ -153,6 +182,8 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, Now: time.Now().UTC().Format(time.RFC3339), } + p.explore(r, &d) + got, err := m.List(r.Context()) if err != nil { p.log.Error("could not read the mirror", "mirror", m.Addr(), "err", err) @@ -226,3 +257,66 @@ func plural(n int, one, many string) string { } return strconv.Itoa(n) + " " + many } + +// explore fills in the registry half of the page: a search, or one repository. +// +// A failure here is reported in its own line rather than as the page's error. +// The registry is somebody else's server on the other side of the internet, and +// it being down says nothing about the mirror — which is the thing this page is +// actually for, and which must keep working when Hugging Face does not. +func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) { + if p.registry == nil { + return + } + d.HasRegistry = true + d.Query = strings.TrimSpace(r.URL.Query().Get("q")) + + if repo := strings.TrimSpace(r.URL.Query().Get("repo")); repo != "" { + got, err := p.registry.Repo(r.Context(), repo) + if err != nil { + d.RegistryErr = err.Error() + return + } + d.Repo = &got + for _, f := range got.Files { + if !f.IsWeights() { + // Thirty files of which four are worth gigabytes. The rest are + // a README and a tokenizer, and listing them buries the ones + // somebody came for. + continue + } + d.RepoFiles = append(d.RepoFiles, repoFile{ + File: f, + Source: got.DownloadURL("", f.Path), + Model: defaultModelName(got.ID), + // The revision, not the date: two fetches of the same commit + // land in the same place, and a repository re-uploaded under + // the same branch lands somewhere else. + Version: got.ShortRevision(), + }) + } + return + } + + if d.Query != "" { + got, err := p.registry.Search(r.Context(), d.Query, 20) + if err != nil { + d.RegistryErr = err.Error() + return + } + d.Results = got + } +} + +// defaultModelName is the catalog name a repository suggests. +// +// The last element, lowercased, with the quantisation marker dropped: the owner +// is not part of what anybody calls the model, and "-GGUF" describes the file +// format rather than the model, which is already clear from the file being +// fetched. +func defaultModelName(repoID string) string { + name := mirror.LastPathElement(repoID) + name = strings.TrimSuffix(name, "-GGUF") + name = strings.TrimSuffix(name, "-gguf") + return strings.ToLower(name) +} diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index b6113b0..4b91cc9 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -14,6 +14,7 @@ import ( "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/account" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" @@ -85,12 +86,17 @@ func (f *fakeMirror) Remove(_ context.Context, model, version string) ([]string, // mirrorPage mounts the page for an admin, or for somebody who is not one. func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux { + return mirrorPageWith(t, m, nil, admin) +} + +// mirrorPageWith mounts it with a registry behind the explorer. +func mirrorPageWith(t *testing.T, m MirrorAdmin, reg Registry, admin bool) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}} p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) mux := http.NewServeMux() - p.RegisterAdminMirror(mux, m) + p.RegisterAdminMirror(mux, m, reg) return mux } @@ -325,3 +331,107 @@ func TestAFailedRowOffersRetryRatherThanRemove(t *testing.T) { t.Error("retry has to carry the source it should fetch again") } } + +type fakeRegistry struct { + results []hf.Model + repo hf.Repo + err error + asked string +} + +func (f *fakeRegistry) Search(_ context.Context, q string, _ int) ([]hf.Model, error) { + f.asked = q + return f.results, f.err +} + +func (f *fakeRegistry) Repo(_ context.Context, id string) (hf.Repo, error) { + f.asked = id + return f.repo, f.err +} + +func TestSearchingTheRegistryListsRepositories(t *testing.T) { + reg := &fakeRegistry{results: []hf.Model{ + {ID: "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF", Downloads: 295447, PipelineTag: "text-generation"}, + }} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), + "/admin/mirror?q=qwen+coder").Body.String() + + if reg.asked != "qwen coder" { + t.Errorf("the query did not arrive: %q", reg.asked) + } + if !strings.Contains(body, "Qwen2.5-Coder-7B-Instruct-GGUF") { + t.Errorf("the result is missing; got %s", body) + } +} + +func TestOnlyWeightFilesAreOffered(t *testing.T) { + reg := &fakeRegistry{repo: hf.Repo{ + ID: "Qwen/X", Revision: "abc123def456", + Files: []hf.File{ + {Path: "README.md", Bytes: 900}, + {Path: "model.gguf", Bytes: 1117320736, SHA256: "6a1a2eb6d15622bf3c96857206351ba97e1af16c30d7a74ee38970e434e9407e"}, + }, + }} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), + "/admin/mirror?repo=Qwen/X").Body.String() + + // A repository is mostly a README and a tokenizer; listing those buries + // the four files somebody came for. + if strings.Contains(body, "README.md") { + t.Error("documentation should not be offered for mirroring") + } + if !strings.Contains(body, "model.gguf") || !strings.Contains(body, "1.0 GiB") { + t.Errorf("the weights and their size should be listed; got %s", body) + } +} + +func TestTheFetchButtonPinsTheRevisionAndSuggestsANames(t *testing.T) { + reg := &fakeRegistry{repo: hf.Repo{ + ID: "Qwen/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abc123def456789", + Files: []hf.File{{Path: "model.gguf", Bytes: 10}}, + }} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), + "/admin/mirror?repo=Qwen/Qwen2.5-1.5B-Instruct-GGUF").Body.String() + + // Pinned, so what arrives is what was listed even if the branch moves. + if !strings.Contains(body, "/resolve/abc123def456789/model.gguf") { + t.Errorf("the source should pin the revision; got %s", body) + } + // The owner is not part of what anybody calls the model, and -GGUF + // describes the file rather than the model. + if !strings.Contains(body, `value="qwen2.5-1.5b-instruct"`) { + t.Error("the suggested name should drop the owner and the format marker") + } + if !strings.Contains(body, `value="abc123de"`) { + t.Error("the version should default to the revision that was listed") + } +} + +func TestARegistryThatIsDownDoesNotBreakTheMirrorPage(t *testing.T) { + reg := &fakeRegistry{err: errors.New("reaching the model registry: timeout")} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{models: []mirror.Status{{ + Model: "held", Version: "v", File: "f", State: mirror.Present, + }}}, reg, true), "/admin/mirror?q=qwen").Body.String() + + // The registry is somebody else's server. It being down says nothing about + // the mirror, which is what this page is for. + if !strings.Contains(body, "The registry:") { + t.Error("the registry failure should be reported on its own line") + } + if !strings.Contains(body, "held") { + t.Error("what the mirror holds must still be listed") + } +} + +func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) { + body := mirrorGet(t, mirrorPage(t, &fakeMirror{}, true), "/admin/mirror").Body.String() + + // A deployment with no way out to the internet gets no search box, rather + // than one that can only time out. + if strings.Contains(body, "Search Hugging Face") { + t.Error("the explorer should be absent without a registry") + } + if !strings.Contains(body, "Fetch a model") { + t.Error("fetching by hand must still be there") + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index dd103ee..e435d84 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -86,7 +86,72 @@ catalog have to spell the same file the same way — two spellings is what makes file that is in use look unreferenced.

{{else}}

The mirror is reachable and holds nothing yet.

{{end}} +{{if .HasRegistry}} +

Find a model

+
+ + +
+ +{{if .RegistryErr}} +{{/* Its own line, not the page's error: the registry is somebody else's server + and it being down says nothing about the mirror, which is what this page is + for and has to keep working when Hugging Face does not. */}} +

The registry: {{.RegistryErr}}

+{{end}} + +{{if .Results}} + +{{range .Results}} + + + + +{{end}} +
RepositoryDownloadsTask
{{.ID}}{{if .Gated.Is}} gated{{end}}{{compact .Downloads}}{{or .PipelineTag "—"}}files
+

A gated repository needs its terms accepted with a Hugging Face +account, and this browses anonymously — the files will list, and fetching one +will fail at the mirror. Download it by hand and put it in the mirror +directory, or give the mirror a token.

+{{else if .Query}}{{if not .RegistryErr}} +

Nothing matched “{{.Query}}”.

+{{end}}{{end}} + +{{if .Repo}} +

{{.Repo.ID}} at {{.Repo.ShortRevision}}

+{{if .RepoFiles}} + + +{{range .RepoFiles}} + + + + +{{end}} +
FileSizePublished checksum
{{.Path}}{{bytes .Bytes}}{{if .SHA256}}{{slice .SHA256 0 12}}{{else}}—{{end}} +
+ + + + + + +
+
+

Only the weights are listed; a repository is mostly a README and +a tokenizer. The source URL is pinned to {{.Repo.ShortRevision}} +rather than to a branch, so what arrives is what is listed here even if the +branch moves in between — and the checksum above is what the registry publishes, +to compare with what the mirror computes once it has the file.

+{{else}} +

No weight files in {{.Repo.ID}}.

+{{end}} +{{end}} +{{end}} +

Fetch a model

+

By hand, for anything the search does not reach.