diff --git a/CHANGELOG.md b/CHANGELOG.md
index afd6084..388a44b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,33 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
node with two kinds of GPU, a shard could get a different card from the one it
was given.
+### A model of many files, mirrored whole for vLLM
+
+vLLM loads a directory — shards, config.json, the tokenizer — and does not
+read GGUF. Mirroring one could go wrong several ways: the explorer offered the
+shards alone, a node could be handed a version still downloading or missing a
+shard, and a repository's page given as a model's source was fetched as one
+file of HTML.
+
+- **The explorer fetches a safetensors set with what the engine reads beside
+ it**: config.json, generation_config.json, the tokenizer files, the shard
+ index, the quantisation config. Never code: a `.py` would be the engine
+ running what the repository ships, which is to be chosen on purpose.
+- **A version is offered to nodes only whole.** The mirror refuses its
+ manifest (409, with what is missing) while a file is still downloading, or
+ while a shard its `model.safetensors.index.json` names, or a part of a
+ split (`-00002-of-00004`), is absent. A node reports "not complete on the
+ mirror yet" and is tried again.
+- **Checksums go with the files.** The mirror keeps each file's sha256 beside
+ it and the manifest carries it; a node checks every file of a set against
+ it before using the set, and fetches one that differs again.
+- **`llmgw mirror add` takes several URLs** into one version.
+- **A Hugging Face repository page is refused as a source**, saying to mirror
+ it or to give the repository id for vLLM to fetch itself.
+
+With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for
+such a model, as before.
+
### Signing up with an address that has an account
On PostgreSQL, signing up with an address already registered showed
diff --git a/internal/agent/weights/set.go b/internal/agent/weights/set.go
index aaba4e1..e0ebf76 100644
--- a/internal/agent/weights/set.go
+++ b/internal/agent/weights/set.go
@@ -2,6 +2,8 @@ package weights
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -26,8 +28,9 @@ type manifest struct {
Model string `json:"model"`
Version string `json:"version"`
Files []struct {
- Name string `json:"name"`
- Bytes int64 `json:"bytes"`
+ Name string `json:"name"`
+ Bytes int64 `json:"bytes"`
+ SHA256 string `json:"sha256"`
} `json:"files"`
Bytes int64 `json:"bytes"`
}
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
}
}
+ // Each file against the checksum the mirror took when it fetched it,
+ // before the set is called complete: a shard damaged on the way loads as
+ // a model that answers nonsense, not as an error.
+ for _, file := range m.Files {
+ if file.SHA256 == "" {
+ continue
+ }
+ target := filepath.Join(dir, file.Name)
+ got, err := fileSHA256(target)
+ if err != nil {
+ return "", err
+ }
+ if !strings.EqualFold(got, file.SHA256) {
+ _ = os.Remove(target)
+ return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
+ source, file.Name, got, file.SHA256)
+ }
+ }
+
if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
return "", err
}
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
}
defer resp.Body.Close()
+ if resp.StatusCode == http.StatusConflict {
+ // Still downloading on the mirror, or a shard missing: said in the
+ // mirror's words, which name what is not there yet.
+ why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+ return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
+ }
if resp.StatusCode != http.StatusOK {
return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
}
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
}
return m, nil
}
+
+func fileSHA256(path string) (string, error) {
+ f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+ return hex.EncodeToString(h.Sum(nil)), nil
+}
diff --git a/internal/agent/weights/set_test.go b/internal/agent/weights/set_test.go
index 4a1e167..93e9063 100644
--- a/internal/agent/weights/set_test.go
+++ b/internal/agent/weights/set_test.go
@@ -1,6 +1,8 @@
package weights
import (
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
}
}
}
+
+func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
+ good := "the shard as mirrored"
+ sum := sha256.Sum256([]byte(good))
+ served := "the shard damaged on the way"
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/") {
+ fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
+ len(served), hex.EncodeToString(sum[:]))
+ return
+ }
+ _, _ = w.Write([]byte(served))
+ }))
+ t.Cleanup(srv.Close)
+ f := &Fetcher{CacheDir: t.TempDir()}
+ if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
+ t.Fatalf("a damaged shard was accepted: %v", err)
+ }
+ served = good
+ // The size differs, so it is fetched again; this time it matches.
+ if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
+ t.Fatalf("the shard as mirrored: %v", err)
+ }
+}
+
+func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
+ }))
+ t.Cleanup(srv.Close)
+ f := &Fetcher{CacheDir: t.TempDir()}
+ _, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
+ if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
+ !strings.Contains(err.Error(), "model-00002-of-00002") {
+ t.Fatalf("got %v", err)
+ }
+}
diff --git a/internal/cli/mirror.go b/internal/cli/mirror.go
index 089dc35..d43165a 100644
--- a/internal/cli/mirror.go
+++ b/internal/cli/mirror.go
@@ -2,6 +2,7 @@ package cli
import (
"encoding/json"
+ "errors"
"fmt"
"log/slog"
"net"
@@ -56,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
var model, version, as string
c := &cobra.Command{
- Use: "add ",
+ Use: "add ...",
Short: "Download a model into the mirror",
Long: "Downloads to a temporary name and renames when complete, so an\n" +
"interrupted download never leaves something a catalog could be pointed\n" +
"at. Reports the checksum: a mirror is the one place in this system where\n" +
"a corrupted file is copied to every node rather than caught once.\n\n" +
+ "Several URLs go into one version, as the files of one model: the shards\n" +
+ "of a safetensors set with its config.json and tokenizer, for vLLM. The\n" +
+ "version is offered to nodes only once it is whole — nothing still\n" +
+ "downloading, every shard its index names present — and its source is\n" +
+ "the version, ending in a slash.\n\n" +
"Refuses to replace a version that is already here. Add a new one beside\n" +
"it instead — that is what makes the old one still available when the new\n" +
"one turns out to be worse.",
- Args: cobra.ExactArgs(1),
+ Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
- source := args[0]
- file := as
- if file == "" {
- file = mirror.LastPathElement(source)
- }
if model == "" || version == "" {
return fmt.Errorf("--model and --version are required: they are the layout")
}
-
- e := mirror.Entry{Model: model, Version: version, File: file}
- out := cmd.OutOrStdout()
- fmt.Fprintf(out, "fetching %s\n", source)
-
- added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
- if total > 0 {
- fmt.Fprintf(out, " %s of %s (%d%%)\n",
- fileSize(done), fileSize(total), done*100/total)
- return
- }
- fmt.Fprintf(out, " %s so far\n", fileSize(done))
- })
- if err != nil {
- return err
+ if as != "" && len(args) > 1 {
+ return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args))
}
- fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n",
- added.Entry.Model, added.Entry.Version, added.Entry.File,
- fileSize(added.Entry.Bytes), added.SHA256)
- fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n",
- added.Entry.URL("http://:8081"))
+ out := cmd.OutOrStdout()
+ var last mirror.Added
+ for _, source := range args {
+ file := as
+ if file == "" {
+ file = mirror.LastPathElement(source)
+ }
+ e := mirror.Entry{Model: model, Version: version, File: file}
+ fmt.Fprintf(out, "fetching %s\n", source)
+ added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
+ if total > 0 {
+ fmt.Fprintf(out, " %s of %s (%d%%)\n",
+ fileSize(done), fileSize(total), done*100/total)
+ return
+ }
+ fmt.Fprintf(out, " %s so far\n", fileSize(done))
+ })
+ if err != nil {
+ return err
+ }
+ fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n",
+ added.Entry.Model, added.Entry.Version, added.Entry.File,
+ fileSize(added.Entry.Bytes), added.SHA256)
+ last = added
+ }
+ src := last.Entry.URL("http://:8081")
+ if len(args) > 1 {
+ src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://:8081"), "/") + "/"
+ if m, err := mirror.ManifestFor(*dir, model, version); err != nil {
+ fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err)
+ } else {
+ fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes))
+ }
+ }
+ fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", src)
return nil
},
}
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well")
- c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted")
+ c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted")
return c
}
@@ -405,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler {
return
}
m, err := mirror.ManifestFor(root, model, version)
- if err != nil {
+ switch {
+ case errors.Is(err, mirror.ErrIncomplete):
+ // Said, not hidden behind a 404: a node asking now is told why
+ // it cannot have the version yet, and tries again later.
+ w.Header().Set("Retry-After", "60")
+ http.Error(w, err.Error(), http.StatusConflict)
+ return
+ case err != nil:
http.NotFound(w, r)
return
}
diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go
index 99afffb..66bea23 100644
--- a/internal/control/catalog/catalog.go
+++ b/internal/control/catalog/catalog.go
@@ -333,6 +333,13 @@ func (m Model) validate() error {
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
+ if repo, ok := repositoryPage(m.Source); ok {
+ // A page, not weights: a node fetches the HTML as one file and the
+ // engine fails at start, on a path that looks right.
+ return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+
+ "mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself",
+ m.Name, m.Source, repo)
+ }
if err := validateAdapters(m); err != nil {
return err
}
@@ -719,3 +726,21 @@ func (m Model) engineAdapters() []engines.Adapter {
}
return out
}
+
+// repositoryPage reports whether source is a Hugging Face repository's page
+// (https://huggingface.co/org/repo, or one of its tree views) rather than a
+// file under /resolve/, and the repository's id.
+func repositoryPage(source string) (string, bool) {
+ for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
+ rest, ok := strings.CutPrefix(source, host)
+ if !ok {
+ continue
+ }
+ parts := strings.Split(strings.Trim(rest, "/"), "/")
+ if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
+ return "", false
+ }
+ return parts[0] + "/" + parts[1], true
+ }
+ return "", false
+}
diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go
index dcc9c9c..6829d78 100644
--- a/internal/control/catalog/catalog_test.go
+++ b/internal/control/catalog/catalog_test.go
@@ -347,3 +347,22 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
}
}
}
+
+func TestARepositoryPageIsNotASource(t *testing.T) {
+ for _, src := range []string{
+ "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ",
+ "https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main",
+ } {
+ m := model("awq")
+ m.Source = src
+ _, err := New([]Model{m})
+ if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) {
+ t.Errorf("%s: %v", src, err)
+ }
+ }
+ m := model("file")
+ m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf"
+ if _, err := New([]Model{m}); err != nil {
+ t.Errorf("a file under /resolve/ is a source: %v", err)
+ }
+}
diff --git a/internal/hf/hf.go b/internal/hf/hf.go
index cd8e647..db663f1 100644
--- a/internal/hf/hf.go
+++ b/internal/hf/hf.go
@@ -258,6 +258,41 @@ func (f File) IsWeights() bool {
return false
}
+// companionNames are the files beside safetensors weights that an engine
+// loading the directory reads: the model's shape, its tokenizer, how it was
+// quantised, and which shard holds which tensor.
+var companionNames = map[string]bool{
+ "config.json": true, "generation_config.json": true,
+ "tokenizer.json": true, "tokenizer_config.json": true, "tokenizer.model": true,
+ "special_tokens_map.json": true, "added_tokens.json": true,
+ "vocab.json": true, "vocab.txt": true, "merges.txt": true,
+ "chat_template.jinja": true, "chat_template.json": true,
+ "preprocessor_config.json": true, "processor_config.json": true,
+ "quantize_config.json": true, "quant_config.json": true,
+}
+
+// Companions are the files of a repository an engine reads beside a
+// safetensors set: config.json and the tokenizer, the shard index. Only
+// those at the top: the directory an engine is given is one level deep.
+//
+// Never code (a .py): a model that needs its own Python to load asks the
+// engine to run what the repository ships, which is a decision for whoever
+// runs the engine, made on purpose, not a file swept in with the tokenizer.
+func Companions(files []File) []File {
+ var out []File
+ for _, f := range files {
+ name := f.Path
+ if strings.Contains(name, "/") {
+ continue
+ }
+ if companionNames[name] || strings.HasSuffix(name, ".safetensors.index.json") ||
+ strings.HasSuffix(name, ".tiktoken") {
+ out = append(out, f)
+ }
+ }
+ return out
+}
+
// Repo is a repository's current revision and what is in it.
type Repo struct {
ID string
diff --git a/internal/mirror/add.go b/internal/mirror/add.go
index fbe10fe..479e10a 100644
--- a/internal/mirror/add.go
+++ b/internal/mirror/add.go
@@ -94,6 +94,13 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
_ = os.Remove(part)
return Added{}, err
}
+ // The checksum beside the file, before the file is in place: a manifest
+ // listing the file then always has its checksum to hand on, and a node
+ // can tell a file damaged on the way from the one that was mirrored.
+ if err := writeChecksum(final, sum); err != nil {
+ _ = os.Remove(part)
+ return Added{}, err
+ }
if err := os.Rename(part, final); err != nil {
return Added{}, err
}
@@ -146,6 +153,9 @@ func Remove(root string, e Entry) error {
if err := os.Remove(path); err != nil {
return err
}
+ // Its checksum goes with it; left, it would keep the directory from
+ // being empty, and so from going.
+ _ = os.Remove(checksumPath(path))
versionDir := filepath.Dir(path)
if empty, _ := isEmpty(versionDir); empty {
_ = os.Remove(versionDir)
diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go
index a76895a..8b154e8 100644
--- a/internal/mirror/mirror.go
+++ b/internal/mirror/mirror.go
@@ -16,12 +16,15 @@ package mirror
import (
"crypto/sha256"
"encoding/hex"
+ "encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
+ "regexp"
"sort"
+ "strconv"
"strings"
)
@@ -91,7 +94,7 @@ func List(root string) ([]Entry, error) {
continue
}
for _, f := range files {
- if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) {
+ if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) || strings.HasPrefix(f.Name(), ".") {
continue
}
info, err := f.Info()
@@ -169,6 +172,31 @@ func normalizeURL(u string) string {
const partSuffix = ".part"
+// checksumPath is where a file's sha256 is kept: beside it, hidden, so no
+// listing takes it for a file of the model and no name a person gives a
+// file (which may not start with a dot) can be the same.
+func checksumPath(file string) string {
+ return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".sha256")
+}
+
+func writeChecksum(file, sum string) error {
+ tmp := checksumPath(file) + partSuffix
+ if err := os.WriteFile(tmp, []byte(sum+"\n"), 0o644); err != nil { //nolint:gosec // a checksum, for anyone to read
+ return err
+ }
+ return os.Rename(tmp, checksumPath(file))
+}
+
+// readChecksum is a file's recorded sha256, or "" for a file put in place by
+// hand, which has none.
+func readChecksum(file string) string {
+ b, err := os.ReadFile(checksumPath(file)) //nolint:gosec // beside a file in the mirror
+ if err != nil {
+ return ""
+ }
+ return strings.TrimSpace(string(b))
+}
+
// sha256File is the checksum of a file on disk.
//
// Reported after a download so an operator has something to compare against
@@ -205,8 +233,16 @@ type Manifest struct {
type ManifestFile struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
+ // SHA256 is the file's checksum when the mirror fetched it, for a node
+ // to check what it downloaded; empty for a file put there by hand.
+ SHA256 string `json:"sha256,omitempty"`
}
+// ErrIncomplete is a version not yet whole: a download still running, or a
+// shard its own index names missing. Its manifest is not given out, so no
+// node starts a model from part of it.
+var ErrIncomplete = errors.New("this version is not complete")
+
// ManifestFor lists the files of one model version.
//
// Built from the directory rather than from a file written alongside it. A
@@ -219,15 +255,79 @@ func ManifestFor(root, model, version string) (Manifest, error) {
return Manifest{}, err
}
m := Manifest{Model: model, Version: version}
+ dir := filepath.Join(root, model, version)
for _, e := range entries {
if e.Model != model || e.Version != version {
continue
}
- m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes})
+ m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes,
+ SHA256: readChecksum(filepath.Join(dir, e.File))})
m.Bytes += e.Bytes
}
if len(m.Files) == 0 {
return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist)
}
+ if err := complete(dir, m.Files); err != nil {
+ return Manifest{}, fmt.Errorf("%s/%s: %w: %v", model, version, ErrIncomplete, err)
+ }
return m, nil
}
+
+// complete says whether a version's files are the whole of it: nothing still
+// downloading, every shard a safetensors index maps a tensor to, and every
+// part of a split ("-00002-of-00004").
+//
+// From what the files say of each other rather than from a list written when
+// the fetch began, so a version filled by hand is held to the same.
+func complete(dir string, files []ManifestFile) error {
+ if parts, _ := filepath.Glob(filepath.Join(dir, "*"+partSuffix)); len(parts) > 0 {
+ var names []string
+ for _, p := range parts {
+ if b := filepath.Base(p); !strings.HasPrefix(b, ".") {
+ names = append(names, strings.TrimSuffix(b, partSuffix))
+ }
+ }
+ if len(names) > 0 {
+ return fmt.Errorf("still downloading %s", strings.Join(names, ", "))
+ }
+ }
+ have := map[string]bool{}
+ for _, f := range files {
+ have[f.Name] = true
+ }
+ var missing []string
+ want := map[string]bool{}
+ for _, f := range files {
+ if strings.HasSuffix(f.Name, ".safetensors.index.json") {
+ var idx struct {
+ WeightMap map[string]string `json:"weight_map"`
+ }
+ b, err := os.ReadFile(filepath.Join(dir, f.Name)) //nolint:gosec // a file in the mirror
+ if err != nil || json.Unmarshal(b, &idx) != nil {
+ return fmt.Errorf("%s cannot be read", f.Name)
+ }
+ for _, shard := range idx.WeightMap {
+ want[shard] = true
+ }
+ }
+ if m := splitPart.FindStringSubmatch(f.Name); m != nil {
+ total, _ := strconv.Atoi(m[3])
+ for i := 1; i <= total; i++ {
+ want[fmt.Sprintf("%s-%0*d-of-%s%s", m[1], len(m[2]), i, m[3], m[4])] = true
+ }
+ }
+ }
+ for name := range want {
+ if !have[name] {
+ missing = append(missing, name)
+ }
+ }
+ if len(missing) > 0 {
+ sort.Strings(missing)
+ return fmt.Errorf("missing %s", strings.Join(missing, ", "))
+ }
+ return nil
+}
+
+// splitPart matches one part of a split: prefix, part, total, extension.
+var splitPart = regexp.MustCompile(`^(.+)-(\d{5})-of-(\d{5})(\.[A-Za-z0-9]+)$`)
diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go
index 0341ac8..477aaaa 100644
--- a/internal/mirror/mirror_test.go
+++ b/internal/mirror/mirror_test.go
@@ -177,3 +177,63 @@ func TestANameThatLeavesTheMirrorIsRefused(t *testing.T) {
}
}
}
+
+func TestAVersionIsGivenOutOnlyWhole(t *testing.T) {
+ root := t.TempDir()
+ dir := filepath.Join(root, "m", "v1")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ write := func(name, body string) {
+ t.Helper()
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ write("config.json", "{}")
+ write("model.safetensors.index.json",
+ `{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}`)
+ write("model-00001-of-00002.safetensors", "aa")
+ write("model-00002-of-00002.safetensors.part", "b")
+
+ if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "still downloading") {
+ t.Fatalf("a version with a shard downloading was given out: %v", err)
+ }
+ // The download failed and its .part went: the index still names it.
+ _ = os.Remove(filepath.Join(dir, "model-00002-of-00002.safetensors.part"))
+ if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) ||
+ !strings.Contains(err.Error(), "missing model-00002-of-00002.safetensors") {
+ t.Fatalf("a version missing a shard its index names was given out: %v", err)
+ }
+ write("model-00002-of-00002.safetensors", "bb")
+ m, err := ManifestFor(root, "m", "v1")
+ if err != nil || len(m.Files) != 4 {
+ t.Fatalf("the whole version: %+v, %v", m, err)
+ }
+
+ // A split GGUF with no index is held to its own numbering.
+ gg := filepath.Join(root, "g", "v1")
+ _ = os.MkdirAll(gg, 0o755)
+ _ = os.WriteFile(filepath.Join(gg, "q-00001-of-00003.gguf"), []byte("x"), 0o644)
+ _ = os.WriteFile(filepath.Join(gg, "q-00003-of-00003.gguf"), []byte("x"), 0o644)
+ if _, err := ManifestFor(root, "g", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "q-00002-of-00003.gguf") {
+ t.Fatalf("a split missing its second part was given out: %v", err)
+ }
+}
+
+func TestAMirroredFilesChecksumIsHandedOnAndHiddenFromListings(t *testing.T) {
+ root := t.TempDir()
+ e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
+ added, err := Add(context.Background(), root, serving(t, "weights"), e, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ entries, _ := List(root)
+ if len(entries) != 1 {
+ t.Errorf("the checksum was listed as a file of the model: %+v", entries)
+ }
+ m, err := ManifestFor(root, "m", "v1")
+ if err != nil || m.Files[0].SHA256 != added.SHA256 || added.SHA256 == "" {
+ t.Errorf("manifest %+v, checksum %q: %v", m, added.SHA256, err)
+ }
+}
diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go
index 71dd736..4df233d 100644
--- a/internal/web/admin_mirror.go
+++ b/internal/web/admin_mirror.go
@@ -60,6 +60,9 @@ type repoSet struct {
// A shard set goes as one action — a fifteen-shard model is not fifteen
// decisions, it is one.
Sources []repoSource
+ // Companions are the files fetched with a safetensors set that are not
+ // weights: config.json, the tokenizer, the shard index.
+ Companions []string
// Model and Version are what the fetch form starts with. Editable, because
// the name a catalog entry uses is the operator's to choose and the
// registry's idea of it is only a good guess.
@@ -778,6 +781,17 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
}
+ // Safetensors are not a model on their own: vLLM loads a directory,
+ // and reads its config and tokenizer before any weight. Fetched as
+ // part of the set, or the set arrives as shards nothing can load.
+ if strings.HasSuffix(strings.ToLower(set.Name), ".safetensors") {
+ for _, f := range hf.Companions(got.Files) {
+ rs.Sources = append(rs.Sources,
+ repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
+ rs.Companions = append(rs.Companions, f.Path)
+ rs.Bytes += f.Bytes
+ }
+ }
d.RepoSets = append(d.RepoSets, rs)
}
p.probeKinds(r.Context(), d.RepoSets)
diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go
index c32c63e..6200954 100644
--- a/internal/web/admin_mirror_test.go
+++ b/internal/web/admin_mirror_test.go
@@ -710,3 +710,37 @@ func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) {
t.Errorf("a started download should be shown where it is followed: %s", body)
}
}
+
+func TestASafetensorsSetIsFetchedWithWhatTheEngineReadsBesideIt(t *testing.T) {
+ // The field: a four-shard AWQ model mirrored as its shards alone, and
+ // vLLM failing at start for want of config.json.
+ reg := &fakeRegistry{repo: hf.Repo{
+ ID: "Qwen/Qwen2.5-32B-Instruct-AWQ", Revision: "abc123def456",
+ Files: []hf.File{
+ {Path: "model-00001-of-00002.safetensors", Bytes: 4_000_000_000},
+ {Path: "model-00002-of-00002.safetensors", Bytes: 3_000_000_000},
+ {Path: "model.safetensors.index.json", Bytes: 90_000},
+ {Path: "config.json", Bytes: 900},
+ {Path: "tokenizer.json", Bytes: 7_000_000},
+ {Path: "tokenizer_config.json", Bytes: 7_000},
+ {Path: "README.md", Bytes: 9_000},
+ {Path: "modeling_custom.py", Bytes: 40_000},
+ {Path: "onnx/config.json", Bytes: 900},
+ },
+ }}
+ body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
+ "/admin/mirror?repo=Qwen/Qwen2.5-32B-Instruct-AWQ").Body.String()
+ for _, want := range []string{"config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors.index.json"} {
+ if !strings.Contains(body, `name="file" value="`+want+`"`) {
+ t.Errorf("%s is not fetched with the set", want)
+ }
+ }
+ for _, not := range []string{"README.md", "modeling_custom.py", "onnx/config.json"} {
+ if strings.Contains(body, `name="file" value="`+not+`"`) {
+ t.Errorf("%s was swept in with the set", not)
+ }
+ }
+ if !strings.Contains(body, "with 4 files the engine reads beside them") {
+ t.Errorf("the companions are not said; got %s", body)
+ }
+}
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index 96e8300..7ebb49e 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -467,6 +467,7 @@ version and the old weights stay where they are.
{{range .RepoSets}}
{{.Name}}
{{if .Shards}}{{.Shards}} shards, fetched together{{end}}
+ {{with .Companions}}with {{len .}} files the engine reads beside them{{end}}
{{if eq .Kind "adapter"}}LoRA adapter · {{.Arch}}
{{else if eq .Kind "model"}}model{{with .Arch}} · {{.}}{{end}}{{end}} |
{{bytes .Bytes}} |
@@ -492,7 +493,8 @@ version and the old weights stay where they are.
{{end}}
Only the weights are listed; a repository is mostly a README and
-a tokenizer. Shards of one file are one row and one button — a fifteen-shard
+a tokenizer. A safetensors set is fetched with the files vLLM reads beside it —
+config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard
model is one decision, not fifteen. Two separate files are two rows because they
are usually alternatives: one quantisation or the other, not both.
The source URLs are pinned to