diff --git a/CHANGELOG.md b/CHANGELOG.md index afd6084..388a44b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,33 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### A model of many files, mirrored whole for vLLM + +vLLM loads a directory — shards, config.json, the tokenizer — and does not +read GGUF. Mirroring one could go wrong several ways: the explorer offered the +shards alone, a node could be handed a version still downloading or missing a +shard, and a repository's page given as a model's source was fetched as one +file of HTML. + +- **The explorer fetches a safetensors set with what the engine reads beside + it**: config.json, generation_config.json, the tokenizer files, the shard + index, the quantisation config. Never code: a `.py` would be the engine + running what the repository ships, which is to be chosen on purpose. +- **A version is offered to nodes only whole.** The mirror refuses its + manifest (409, with what is missing) while a file is still downloading, or + while a shard its `model.safetensors.index.json` names, or a part of a + split (`-00002-of-00004`), is absent. A node reports "not complete on the + mirror yet" and is tried again. +- **Checksums go with the files.** The mirror keeps each file's sha256 beside + it and the manifest carries it; a node checks every file of a set against + it before using the set, and fetches one that differs again. +- **`llmgw mirror add` takes several URLs** into one version. +- **A Hugging Face repository page is refused as a source**, saying to mirror + it or to give the repository id for vLLM to fetch itself. + +With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for +such a model, as before. + ### Signing up with an address that has an account On PostgreSQL, signing up with an address already registered showed diff --git a/internal/agent/weights/set.go b/internal/agent/weights/set.go index aaba4e1..e0ebf76 100644 --- a/internal/agent/weights/set.go +++ b/internal/agent/weights/set.go @@ -2,6 +2,8 @@ package weights import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io" @@ -26,8 +28,9 @@ type manifest struct { Model string `json:"model"` Version string `json:"version"` Files []struct { - Name string `json:"name"` - Bytes int64 `json:"bytes"` + Name string `json:"name"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` } `json:"files"` Bytes int64 `json:"bytes"` } @@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error) } } + // Each file against the checksum the mirror took when it fetched it, + // before the set is called complete: a shard damaged on the way loads as + // a model that answers nonsense, not as an error. + for _, file := range m.Files { + if file.SHA256 == "" { + continue + } + target := filepath.Join(dir, file.Name) + got, err := fileSHA256(target) + if err != nil { + return "", err + } + if !strings.EqualFold(got, file.SHA256) { + _ = os.Remove(target) + return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again", + source, file.Name, got, file.SHA256) + } + } + if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil { return "", err } @@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err) } defer resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + // Still downloading on the mirror, or a shard missing: said in the + // mirror's words, which name what is not there yet. + why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why))) + } if resp.StatusCode != http.StatusOK { return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode) } @@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err } return m, nil } + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/agent/weights/set_test.go b/internal/agent/weights/set_test.go index 4a1e167..93e9063 100644 --- a/internal/agent/weights/set_test.go +++ b/internal/agent/weights/set_test.go @@ -1,6 +1,8 @@ package weights import ( + "crypto/sha256" + "encoding/hex" "fmt" "net/http" "net/http/httptest" @@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) { } } } + +func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) { + good := "the shard as mirrored" + sum := sha256.Sum256([]byte(good)) + served := "the shard damaged on the way" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`, + len(served), hex.EncodeToString(sum[:])) + return + } + _, _ = w.Write([]byte(served)) + })) + t.Cleanup(srv.Close) + f := &Fetcher{CacheDir: t.TempDir()} + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") { + t.Fatalf("a damaged shard was accepted: %v", err) + } + served = good + // The size differs, so it is fetched again; this time it matches. + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatalf("the shard as mirrored: %v", err) + } +} + +func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict) + })) + t.Cleanup(srv.Close) + f := &Fetcher{CacheDir: t.TempDir()} + _, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") || + !strings.Contains(err.Error(), "model-00002-of-00002") { + t.Fatalf("got %v", err) + } +} diff --git a/internal/cli/mirror.go b/internal/cli/mirror.go index 089dc35..d43165a 100644 --- a/internal/cli/mirror.go +++ b/internal/cli/mirror.go @@ -2,6 +2,7 @@ package cli import ( "encoding/json" + "errors" "fmt" "log/slog" "net" @@ -56,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command { var model, version, as string c := &cobra.Command{ - Use: "add ", + Use: "add ...", Short: "Download a model into the mirror", Long: "Downloads to a temporary name and renames when complete, so an\n" + "interrupted download never leaves something a catalog could be pointed\n" + "at. Reports the checksum: a mirror is the one place in this system where\n" + "a corrupted file is copied to every node rather than caught once.\n\n" + + "Several URLs go into one version, as the files of one model: the shards\n" + + "of a safetensors set with its config.json and tokenizer, for vLLM. The\n" + + "version is offered to nodes only once it is whole — nothing still\n" + + "downloading, every shard its index names present — and its source is\n" + + "the version, ending in a slash.\n\n" + "Refuses to replace a version that is already here. Add a new one beside\n" + "it instead — that is what makes the old one still available when the new\n" + "one turns out to be worse.", - Args: cobra.ExactArgs(1), + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - source := args[0] - file := as - if file == "" { - file = mirror.LastPathElement(source) - } if model == "" || version == "" { return fmt.Errorf("--model and --version are required: they are the layout") } - - e := mirror.Entry{Model: model, Version: version, File: file} - out := cmd.OutOrStdout() - fmt.Fprintf(out, "fetching %s\n", source) - - added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) { - if total > 0 { - fmt.Fprintf(out, " %s of %s (%d%%)\n", - fileSize(done), fileSize(total), done*100/total) - return - } - fmt.Fprintf(out, " %s so far\n", fileSize(done)) - }) - if err != nil { - return err + if as != "" && len(args) > 1 { + return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args)) } - fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n", - added.Entry.Model, added.Entry.Version, added.Entry.File, - fileSize(added.Entry.Bytes), added.SHA256) - fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", - added.Entry.URL("http://:8081")) + out := cmd.OutOrStdout() + var last mirror.Added + for _, source := range args { + file := as + if file == "" { + file = mirror.LastPathElement(source) + } + e := mirror.Entry{Model: model, Version: version, File: file} + fmt.Fprintf(out, "fetching %s\n", source) + added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) { + if total > 0 { + fmt.Fprintf(out, " %s of %s (%d%%)\n", + fileSize(done), fileSize(total), done*100/total) + return + } + fmt.Fprintf(out, " %s so far\n", fileSize(done)) + }) + if err != nil { + return err + } + fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n", + added.Entry.Model, added.Entry.Version, added.Entry.File, + fileSize(added.Entry.Bytes), added.SHA256) + last = added + } + src := last.Entry.URL("http://:8081") + if len(args) > 1 { + src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://:8081"), "/") + "/" + if m, err := mirror.ManifestFor(*dir, model, version); err != nil { + fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err) + } else { + fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes)) + } + } + fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", src) return nil }, } c.Flags().StringVar(&model, "model", "", "the model's name, one directory level") c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well") - c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted") + c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted") return c } @@ -405,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler { return } m, err := mirror.ManifestFor(root, model, version) - if err != nil { + switch { + case errors.Is(err, mirror.ErrIncomplete): + // Said, not hidden behind a 404: a node asking now is told why + // it cannot have the version yet, and tries again later. + w.Header().Set("Retry-After", "60") + http.Error(w, err.Error(), http.StatusConflict) + return + case err != nil: http.NotFound(w, r) return } diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 99afffb..66bea23 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -333,6 +333,13 @@ func (m Model) validate() error { if m.Source == "" { return fmt.Errorf("model %q: source is required", m.Name) } + if repo, ok := repositoryPage(m.Source); ok { + // A page, not weights: a node fetches the HTML as one file and the + // engine fails at start, on a path that looks right. + return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+ + "mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself", + m.Name, m.Source, repo) + } if err := validateAdapters(m); err != nil { return err } @@ -719,3 +726,21 @@ func (m Model) engineAdapters() []engines.Adapter { } return out } + +// repositoryPage reports whether source is a Hugging Face repository's page +// (https://huggingface.co/org/repo, or one of its tree views) rather than a +// file under /resolve/, and the repository's id. +func repositoryPage(source string) (string, bool) { + for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} { + rest, ok := strings.CutPrefix(source, host) + if !ok { + continue + } + parts := strings.Split(strings.Trim(rest, "/"), "/") + if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") { + return "", false + } + return parts[0] + "/" + parts[1], true + } + return "", false +} diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index dcc9c9c..6829d78 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -347,3 +347,22 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) { } } } + +func TestARepositoryPageIsNotASource(t *testing.T) { + for _, src := range []string{ + "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ", + "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main", + } { + m := model("awq") + m.Source = src + _, err := New([]Model{m}) + if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) { + t.Errorf("%s: %v", src, err) + } + } + m := model("file") + m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf" + if _, err := New([]Model{m}); err != nil { + t.Errorf("a file under /resolve/ is a source: %v", err) + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index cd8e647..db663f1 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -258,6 +258,41 @@ func (f File) IsWeights() bool { return false } +// companionNames are the files beside safetensors weights that an engine +// loading the directory reads: the model's shape, its tokenizer, how it was +// quantised, and which shard holds which tensor. +var companionNames = map[string]bool{ + "config.json": true, "generation_config.json": true, + "tokenizer.json": true, "tokenizer_config.json": true, "tokenizer.model": true, + "special_tokens_map.json": true, "added_tokens.json": true, + "vocab.json": true, "vocab.txt": true, "merges.txt": true, + "chat_template.jinja": true, "chat_template.json": true, + "preprocessor_config.json": true, "processor_config.json": true, + "quantize_config.json": true, "quant_config.json": true, +} + +// Companions are the files of a repository an engine reads beside a +// safetensors set: config.json and the tokenizer, the shard index. Only +// those at the top: the directory an engine is given is one level deep. +// +// Never code (a .py): a model that needs its own Python to load asks the +// engine to run what the repository ships, which is a decision for whoever +// runs the engine, made on purpose, not a file swept in with the tokenizer. +func Companions(files []File) []File { + var out []File + for _, f := range files { + name := f.Path + if strings.Contains(name, "/") { + continue + } + if companionNames[name] || strings.HasSuffix(name, ".safetensors.index.json") || + strings.HasSuffix(name, ".tiktoken") { + out = append(out, f) + } + } + return out +} + // Repo is a repository's current revision and what is in it. type Repo struct { ID string diff --git a/internal/mirror/add.go b/internal/mirror/add.go index fbe10fe..479e10a 100644 --- a/internal/mirror/add.go +++ b/internal/mirror/add.go @@ -94,6 +94,13 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En _ = os.Remove(part) return Added{}, err } + // The checksum beside the file, before the file is in place: a manifest + // listing the file then always has its checksum to hand on, and a node + // can tell a file damaged on the way from the one that was mirrored. + if err := writeChecksum(final, sum); err != nil { + _ = os.Remove(part) + return Added{}, err + } if err := os.Rename(part, final); err != nil { return Added{}, err } @@ -146,6 +153,9 @@ func Remove(root string, e Entry) error { if err := os.Remove(path); err != nil { return err } + // Its checksum goes with it; left, it would keep the directory from + // being empty, and so from going. + _ = os.Remove(checksumPath(path)) versionDir := filepath.Dir(path) if empty, _ := isEmpty(versionDir); empty { _ = os.Remove(versionDir) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index a76895a..8b154e8 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -16,12 +16,15 @@ package mirror import ( "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" "os" "path/filepath" + "regexp" "sort" + "strconv" "strings" ) @@ -91,7 +94,7 @@ func List(root string) ([]Entry, error) { continue } for _, f := range files { - if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) { + if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) || strings.HasPrefix(f.Name(), ".") { continue } info, err := f.Info() @@ -169,6 +172,31 @@ func normalizeURL(u string) string { const partSuffix = ".part" +// checksumPath is where a file's sha256 is kept: beside it, hidden, so no +// listing takes it for a file of the model and no name a person gives a +// file (which may not start with a dot) can be the same. +func checksumPath(file string) string { + return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".sha256") +} + +func writeChecksum(file, sum string) error { + tmp := checksumPath(file) + partSuffix + if err := os.WriteFile(tmp, []byte(sum+"\n"), 0o644); err != nil { //nolint:gosec // a checksum, for anyone to read + return err + } + return os.Rename(tmp, checksumPath(file)) +} + +// readChecksum is a file's recorded sha256, or "" for a file put in place by +// hand, which has none. +func readChecksum(file string) string { + b, err := os.ReadFile(checksumPath(file)) //nolint:gosec // beside a file in the mirror + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + // sha256File is the checksum of a file on disk. // // Reported after a download so an operator has something to compare against @@ -205,8 +233,16 @@ type Manifest struct { type ManifestFile struct { Name string `json:"name"` Bytes int64 `json:"bytes"` + // SHA256 is the file's checksum when the mirror fetched it, for a node + // to check what it downloaded; empty for a file put there by hand. + SHA256 string `json:"sha256,omitempty"` } +// ErrIncomplete is a version not yet whole: a download still running, or a +// shard its own index names missing. Its manifest is not given out, so no +// node starts a model from part of it. +var ErrIncomplete = errors.New("this version is not complete") + // ManifestFor lists the files of one model version. // // Built from the directory rather than from a file written alongside it. A @@ -219,15 +255,79 @@ func ManifestFor(root, model, version string) (Manifest, error) { return Manifest{}, err } m := Manifest{Model: model, Version: version} + dir := filepath.Join(root, model, version) for _, e := range entries { if e.Model != model || e.Version != version { continue } - m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes}) + m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes, + SHA256: readChecksum(filepath.Join(dir, e.File))}) m.Bytes += e.Bytes } if len(m.Files) == 0 { return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist) } + if err := complete(dir, m.Files); err != nil { + return Manifest{}, fmt.Errorf("%s/%s: %w: %v", model, version, ErrIncomplete, err) + } return m, nil } + +// complete says whether a version's files are the whole of it: nothing still +// downloading, every shard a safetensors index maps a tensor to, and every +// part of a split ("-00002-of-00004"). +// +// From what the files say of each other rather than from a list written when +// the fetch began, so a version filled by hand is held to the same. +func complete(dir string, files []ManifestFile) error { + if parts, _ := filepath.Glob(filepath.Join(dir, "*"+partSuffix)); len(parts) > 0 { + var names []string + for _, p := range parts { + if b := filepath.Base(p); !strings.HasPrefix(b, ".") { + names = append(names, strings.TrimSuffix(b, partSuffix)) + } + } + if len(names) > 0 { + return fmt.Errorf("still downloading %s", strings.Join(names, ", ")) + } + } + have := map[string]bool{} + for _, f := range files { + have[f.Name] = true + } + var missing []string + want := map[string]bool{} + for _, f := range files { + if strings.HasSuffix(f.Name, ".safetensors.index.json") { + var idx struct { + WeightMap map[string]string `json:"weight_map"` + } + b, err := os.ReadFile(filepath.Join(dir, f.Name)) //nolint:gosec // a file in the mirror + if err != nil || json.Unmarshal(b, &idx) != nil { + return fmt.Errorf("%s cannot be read", f.Name) + } + for _, shard := range idx.WeightMap { + want[shard] = true + } + } + if m := splitPart.FindStringSubmatch(f.Name); m != nil { + total, _ := strconv.Atoi(m[3]) + for i := 1; i <= total; i++ { + want[fmt.Sprintf("%s-%0*d-of-%s%s", m[1], len(m[2]), i, m[3], m[4])] = true + } + } + } + for name := range want { + if !have[name] { + missing = append(missing, name) + } + } + if len(missing) > 0 { + sort.Strings(missing) + return fmt.Errorf("missing %s", strings.Join(missing, ", ")) + } + return nil +} + +// splitPart matches one part of a split: prefix, part, total, extension. +var splitPart = regexp.MustCompile(`^(.+)-(\d{5})-of-(\d{5})(\.[A-Za-z0-9]+)$`) diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 0341ac8..477aaaa 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -177,3 +177,63 @@ func TestANameThatLeavesTheMirrorIsRefused(t *testing.T) { } } } + +func TestAVersionIsGivenOutOnlyWhole(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "m", "v1") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("config.json", "{}") + write("model.safetensors.index.json", + `{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}`) + write("model-00001-of-00002.safetensors", "aa") + write("model-00002-of-00002.safetensors.part", "b") + + if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "still downloading") { + t.Fatalf("a version with a shard downloading was given out: %v", err) + } + // The download failed and its .part went: the index still names it. + _ = os.Remove(filepath.Join(dir, "model-00002-of-00002.safetensors.part")) + if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) || + !strings.Contains(err.Error(), "missing model-00002-of-00002.safetensors") { + t.Fatalf("a version missing a shard its index names was given out: %v", err) + } + write("model-00002-of-00002.safetensors", "bb") + m, err := ManifestFor(root, "m", "v1") + if err != nil || len(m.Files) != 4 { + t.Fatalf("the whole version: %+v, %v", m, err) + } + + // A split GGUF with no index is held to its own numbering. + gg := filepath.Join(root, "g", "v1") + _ = os.MkdirAll(gg, 0o755) + _ = os.WriteFile(filepath.Join(gg, "q-00001-of-00003.gguf"), []byte("x"), 0o644) + _ = os.WriteFile(filepath.Join(gg, "q-00003-of-00003.gguf"), []byte("x"), 0o644) + if _, err := ManifestFor(root, "g", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "q-00002-of-00003.gguf") { + t.Fatalf("a split missing its second part was given out: %v", err) + } +} + +func TestAMirroredFilesChecksumIsHandedOnAndHiddenFromListings(t *testing.T) { + root := t.TempDir() + e := Entry{Model: "m", Version: "v1", File: "f.gguf"} + added, err := Add(context.Background(), root, serving(t, "weights"), e, nil) + if err != nil { + t.Fatal(err) + } + entries, _ := List(root) + if len(entries) != 1 { + t.Errorf("the checksum was listed as a file of the model: %+v", entries) + } + m, err := ManifestFor(root, "m", "v1") + if err != nil || m.Files[0].SHA256 != added.SHA256 || added.SHA256 == "" { + t.Errorf("manifest %+v, checksum %q: %v", m, added.SHA256, err) + } +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 71dd736..4df233d 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -60,6 +60,9 @@ type repoSet struct { // A shard set goes as one action — a fifteen-shard model is not fifteen // decisions, it is one. Sources []repoSource + // Companions are the files fetched with a safetensors set that are not + // weights: config.json, the tokenizer, the shard index. + Companions []string // Model and Version are what the fetch form starts with. Editable, because // the name a catalog entry uses is the operator's to choose and the // registry's idea of it is only a good guess. @@ -778,6 +781,17 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string rs.Sources = append(rs.Sources, repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path}) } + // Safetensors are not a model on their own: vLLM loads a directory, + // and reads its config and tokenizer before any weight. Fetched as + // part of the set, or the set arrives as shards nothing can load. + if strings.HasSuffix(strings.ToLower(set.Name), ".safetensors") { + for _, f := range hf.Companions(got.Files) { + rs.Sources = append(rs.Sources, + repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path}) + rs.Companions = append(rs.Companions, f.Path) + rs.Bytes += f.Bytes + } + } d.RepoSets = append(d.RepoSets, rs) } p.probeKinds(r.Context(), d.RepoSets) diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index c32c63e..6200954 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -710,3 +710,37 @@ func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) { t.Errorf("a started download should be shown where it is followed: %s", body) } } + +func TestASafetensorsSetIsFetchedWithWhatTheEngineReadsBesideIt(t *testing.T) { + // The field: a four-shard AWQ model mirrored as its shards alone, and + // vLLM failing at start for want of config.json. + reg := &fakeRegistry{repo: hf.Repo{ + ID: "Qwen/Qwen2.5-32B-Instruct-AWQ", Revision: "abc123def456", + Files: []hf.File{ + {Path: "model-00001-of-00002.safetensors", Bytes: 4_000_000_000}, + {Path: "model-00002-of-00002.safetensors", Bytes: 3_000_000_000}, + {Path: "model.safetensors.index.json", Bytes: 90_000}, + {Path: "config.json", Bytes: 900}, + {Path: "tokenizer.json", Bytes: 7_000_000}, + {Path: "tokenizer_config.json", Bytes: 7_000}, + {Path: "README.md", Bytes: 9_000}, + {Path: "modeling_custom.py", Bytes: 40_000}, + {Path: "onnx/config.json", Bytes: 900}, + }, + }} + body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true), + "/admin/mirror?repo=Qwen/Qwen2.5-32B-Instruct-AWQ").Body.String() + for _, want := range []string{"config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors.index.json"} { + if !strings.Contains(body, `name="file" value="`+want+`"`) { + t.Errorf("%s is not fetched with the set", want) + } + } + for _, not := range []string{"README.md", "modeling_custom.py", "onnx/config.json"} { + if strings.Contains(body, `name="file" value="`+not+`"`) { + t.Errorf("%s was swept in with the set", not) + } + } + if !strings.Contains(body, "with 4 files the engine reads beside them") { + t.Errorf("the companions are not said; got %s", body) + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 96e8300..7ebb49e 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -467,6 +467,7 @@ version and the old weights stay where they are.

{{range .RepoSets}} {{.Name}} {{if .Shards}}{{.Shards}} shards, fetched together{{end}} + {{with .Companions}}with {{len .}} files the engine reads beside them{{end}} {{if eq .Kind "adapter"}}LoRA adapter · {{.Arch}} {{else if eq .Kind "model"}}model{{with .Arch}} · {{.}}{{end}}{{end}} {{bytes .Bytes}} @@ -492,7 +493,8 @@ version and the old weights stay where they are.

{{end}}

Only the weights are listed; a repository is mostly a README and -a tokenizer. Shards of one file are one row and one button — a fifteen-shard +a tokenizer. A safetensors set is fetched with the files vLLM reads beside it — +config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard model is one decision, not fifteen. Two separate files are two rows because they are usually alternatives: one quantisation or the other, not both.

The source URLs are pinned to