Mirror a many-file model whole, with its checksums, for vLLM
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m58s

The explorer fetches a safetensors set with its config and tokenizer;
the mirror gives out a version's manifest only once nothing is
downloading and every shard its index names is there, with each file's
sha256, which nodes check; mirror add takes several URLs; a repository
page is refused as a source.

Refs #2 (item 9).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 12:21:04 +02:00
1 parent 8957db30a2
commit 2735c5a318
13 files changed
+465 -34

No files matched your search

+27
View File
@@ -47,6 +47,33 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
node with two kinds of GPU, a shard could get a different card from the one it
was given.
### A model of many files, mirrored whole for vLLM
vLLM loads a directory — shards, config.json, the tokenizer — and does not
read GGUF. Mirroring one could go wrong several ways: the explorer offered the
shards alone, a node could be handed a version still downloading or missing a
shard, and a repository's page given as a model's source was fetched as one
file of HTML.
- **The explorer fetches a safetensors set with what the engine reads beside
it**: config.json, generation_config.json, the tokenizer files, the shard
index, the quantisation config. Never code: a `.py` would be the engine
running what the repository ships, which is to be chosen on purpose.
- **A version is offered to nodes only whole.** The mirror refuses its
manifest (409, with what is missing) while a file is still downloading, or
while a shard its `model.safetensors.index.json` names, or a part of a
split (`-00002-of-00004`), is absent. A node reports "not complete on the
mirror yet" and is tried again.
- **Checksums go with the files.** The mirror keeps each file's sha256 beside
it and the manifest carries it; a node checks every file of a set against
it before using the set, and fetches one that differs again.
- **`llmgw mirror add` takes several URLs** into one version.
- **A Hugging Face repository page is refused as a source**, saying to mirror
it or to give the repository id for vLLM to fetch itself.
With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for
such a model, as before.
### Signing up with an address that has an account
On PostgreSQL, signing up with an address already registered showed
+43 -2
View File
@@ -2,6 +2,8 @@ package weights
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -26,8 +28,9 @@ type manifest struct {
Model string `json:"model"`
Version string `json:"version"`
Files []struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
Name string `json:"name"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
} `json:"files"`
Bytes int64 `json:"bytes"`
}
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
}
}
// Each file against the checksum the mirror took when it fetched it,
// before the set is called complete: a shard damaged on the way loads as
// a model that answers nonsense, not as an error.
for _, file := range m.Files {
if file.SHA256 == "" {
continue
}
target := filepath.Join(dir, file.Name)
got, err := fileSHA256(target)
if err != nil {
return "", err
}
if !strings.EqualFold(got, file.SHA256) {
_ = os.Remove(target)
return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
source, file.Name, got, file.SHA256)
}
}
if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
return "", err
}
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusConflict {
// Still downloading on the mirror, or a shard missing: said in the
// mirror's words, which name what is not there yet.
why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
}
if resp.StatusCode != http.StatusOK {
return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
}
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
}
return m, nil
}
func fileSHA256(path string) (string, error) {
f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
+39
View File
@@ -1,6 +1,8 @@
package weights
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
}
}
}
func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
good := "the shard as mirrored"
sum := sha256.Sum256([]byte(good))
served := "the shard damaged on the way"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
len(served), hex.EncodeToString(sum[:]))
return
}
_, _ = w.Write([]byte(served))
}))
t.Cleanup(srv.Close)
f := &Fetcher{CacheDir: t.TempDir()}
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
t.Fatalf("a damaged shard was accepted: %v", err)
}
served = good
// The size differs, so it is fetched again; this time it matches.
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
t.Fatalf("the shard as mirrored: %v", err)
}
}
func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
}))
t.Cleanup(srv.Close)
f := &Fetcher{CacheDir: t.TempDir()}
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
!strings.Contains(err.Error(), "model-00002-of-00002") {
t.Fatalf("got %v", err)
}
}
+54 -29
View File
@@ -2,6 +2,7 @@ package cli
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
@@ -56,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
var model, version, as string
c := &cobra.Command{
Use: "add <url>",
Use: "add <url>...",
Short: "Download a model into the mirror",
Long: "Downloads to a temporary name and renames when complete, so an\n" +
"interrupted download never leaves something a catalog could be pointed\n" +
"at. Reports the checksum: a mirror is the one place in this system where\n" +
"a corrupted file is copied to every node rather than caught once.\n\n" +
"Several URLs go into one version, as the files of one model: the shards\n" +
"of a safetensors set with its config.json and tokenizer, for vLLM. The\n" +
"version is offered to nodes only once it is whole — nothing still\n" +
"downloading, every shard its index names present — and its source is\n" +
"the version, ending in a slash.\n\n" +
"Refuses to replace a version that is already here. Add a new one beside\n" +
"it instead — that is what makes the old one still available when the new\n" +
"one turns out to be worse.",
Args: cobra.ExactArgs(1),
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
source := args[0]
file := as
if file == "" {
file = mirror.LastPathElement(source)
}
if model == "" || version == "" {
return fmt.Errorf("--model and --version are required: they are the layout")
}
e := mirror.Entry{Model: model, Version: version, File: file}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "fetching %s\n", source)
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
if total > 0 {
fmt.Fprintf(out, " %s of %s (%d%%)\n",
fileSize(done), fileSize(total), done*100/total)
return
}
fmt.Fprintf(out, " %s so far\n", fileSize(done))
})
if err != nil {
return err
if as != "" && len(args) > 1 {
return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args))
}
fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n",
added.Entry.Model, added.Entry.Version, added.Entry.File,
fileSize(added.Entry.Bytes), added.SHA256)
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n",
added.Entry.URL("http://<this-host>:8081"))
out := cmd.OutOrStdout()
var last mirror.Added
for _, source := range args {
file := as
if file == "" {
file = mirror.LastPathElement(source)
}
e := mirror.Entry{Model: model, Version: version, File: file}
fmt.Fprintf(out, "fetching %s\n", source)
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
if total > 0 {
fmt.Fprintf(out, " %s of %s (%d%%)\n",
fileSize(done), fileSize(total), done*100/total)
return
}
fmt.Fprintf(out, " %s so far\n", fileSize(done))
})
if err != nil {
return err
}
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n",
added.Entry.Model, added.Entry.Version, added.Entry.File,
fileSize(added.Entry.Bytes), added.SHA256)
last = added
}
src := last.Entry.URL("http://<this-host>:8081")
if len(args) > 1 {
src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://<this-host>:8081"), "/") + "/"
if m, err := mirror.ManifestFor(*dir, model, version); err != nil {
fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err)
} else {
fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes))
}
}
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", src)
return nil
},
}
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well")
c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted")
c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted")
return c
}
@@ -405,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler {
return
}
m, err := mirror.ManifestFor(root, model, version)
if err != nil {
switch {
case errors.Is(err, mirror.ErrIncomplete):
// Said, not hidden behind a 404: a node asking now is told why
// it cannot have the version yet, and tries again later.
w.Header().Set("Retry-After", "60")
http.Error(w, err.Error(), http.StatusConflict)
return
case err != nil:
http.NotFound(w, r)
return
}
+25
View File
@@ -333,6 +333,13 @@ func (m Model) validate() error {
if m.Source == "" {
return fmt.Errorf("model %q: source is required", m.Name)
}
if repo, ok := repositoryPage(m.Source); ok {
// A page, not weights: a node fetches the HTML as one file and the
// engine fails at start, on a path that looks right.
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+
"mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself",
m.Name, m.Source, repo)
}
if err := validateAdapters(m); err != nil {
return err
}
@@ -719,3 +726,21 @@ func (m Model) engineAdapters() []engines.Adapter {
}
return out
}
// repositoryPage reports whether source is a Hugging Face repository's page
// (https://huggingface.co/org/repo, or one of its tree views) rather than a
// file under /resolve/, and the repository's id.
func repositoryPage(source string) (string, bool) {
for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
rest, ok := strings.CutPrefix(source, host)
if !ok {
continue
}
parts := strings.Split(strings.Trim(rest, "/"), "/")
if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
return "", false
}
return parts[0] + "/" + parts[1], true
}
return "", false
}
+19
View File
@@ -347,3 +347,22 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
}
}
}
func TestARepositoryPageIsNotASource(t *testing.T) {
for _, src := range []string{
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ",
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main",
} {
m := model("awq")
m.Source = src
_, err := New([]Model{m})
if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) {
t.Errorf("%s: %v", src, err)
}
}
m := model("file")
m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf"
if _, err := New([]Model{m}); err != nil {
t.Errorf("a file under /resolve/ is a source: %v", err)
}
}
+35
View File
@@ -258,6 +258,41 @@ func (f File) IsWeights() bool {
return false
}
// companionNames are the files beside safetensors weights that an engine
// loading the directory reads: the model's shape, its tokenizer, how it was
// quantised, and which shard holds which tensor.
var companionNames = map[string]bool{
"config.json": true, "generation_config.json": true,
"tokenizer.json": true, "tokenizer_config.json": true, "tokenizer.model": true,
"special_tokens_map.json": true, "added_tokens.json": true,
"vocab.json": true, "vocab.txt": true, "merges.txt": true,
"chat_template.jinja": true, "chat_template.json": true,
"preprocessor_config.json": true, "processor_config.json": true,
"quantize_config.json": true, "quant_config.json": true,
}
// Companions are the files of a repository an engine reads beside a
// safetensors set: config.json and the tokenizer, the shard index. Only
// those at the top: the directory an engine is given is one level deep.
//
// Never code (a .py): a model that needs its own Python to load asks the
// engine to run what the repository ships, which is a decision for whoever
// runs the engine, made on purpose, not a file swept in with the tokenizer.
func Companions(files []File) []File {
var out []File
for _, f := range files {
name := f.Path
if strings.Contains(name, "/") {
continue
}
if companionNames[name] || strings.HasSuffix(name, ".safetensors.index.json") ||
strings.HasSuffix(name, ".tiktoken") {
out = append(out, f)
}
}
return out
}
// Repo is a repository's current revision and what is in it.
type Repo struct {
ID string
+10
View File
@@ -94,6 +94,13 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
_ = os.Remove(part)
return Added{}, err
}
// The checksum beside the file, before the file is in place: a manifest
// listing the file then always has its checksum to hand on, and a node
// can tell a file damaged on the way from the one that was mirrored.
if err := writeChecksum(final, sum); err != nil {
_ = os.Remove(part)
return Added{}, err
}
if err := os.Rename(part, final); err != nil {
return Added{}, err
}
@@ -146,6 +153,9 @@ func Remove(root string, e Entry) error {
if err := os.Remove(path); err != nil {
return err
}
// Its checksum goes with it; left, it would keep the directory from
// being empty, and so from going.
_ = os.Remove(checksumPath(path))
versionDir := filepath.Dir(path)
if empty, _ := isEmpty(versionDir); empty {
_ = os.Remove(versionDir)
+102 -2
View File
@@ -16,12 +16,15 @@ package mirror
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
)
@@ -91,7 +94,7 @@ func List(root string) ([]Entry, error) {
continue
}
for _, f := range files {
if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) {
if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) || strings.HasPrefix(f.Name(), ".") {
continue
}
info, err := f.Info()
@@ -169,6 +172,31 @@ func normalizeURL(u string) string {
const partSuffix = ".part"
// checksumPath is where a file's sha256 is kept: beside it, hidden, so no
// listing takes it for a file of the model and no name a person gives a
// file (which may not start with a dot) can be the same.
func checksumPath(file string) string {
return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".sha256")
}
func writeChecksum(file, sum string) error {
tmp := checksumPath(file) + partSuffix
if err := os.WriteFile(tmp, []byte(sum+"\n"), 0o644); err != nil { //nolint:gosec // a checksum, for anyone to read
return err
}
return os.Rename(tmp, checksumPath(file))
}
// readChecksum is a file's recorded sha256, or "" for a file put in place by
// hand, which has none.
func readChecksum(file string) string {
b, err := os.ReadFile(checksumPath(file)) //nolint:gosec // beside a file in the mirror
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}
// sha256File is the checksum of a file on disk.
//
// Reported after a download so an operator has something to compare against
@@ -205,8 +233,16 @@ type Manifest struct {
type ManifestFile struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
// SHA256 is the file's checksum when the mirror fetched it, for a node
// to check what it downloaded; empty for a file put there by hand.
SHA256 string `json:"sha256,omitempty"`
}
// ErrIncomplete is a version not yet whole: a download still running, or a
// shard its own index names missing. Its manifest is not given out, so no
// node starts a model from part of it.
var ErrIncomplete = errors.New("this version is not complete")
// ManifestFor lists the files of one model version.
//
// Built from the directory rather than from a file written alongside it. A
@@ -219,15 +255,79 @@ func ManifestFor(root, model, version string) (Manifest, error) {
return Manifest{}, err
}
m := Manifest{Model: model, Version: version}
dir := filepath.Join(root, model, version)
for _, e := range entries {
if e.Model != model || e.Version != version {
continue
}
m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes})
m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes,
SHA256: readChecksum(filepath.Join(dir, e.File))})
m.Bytes += e.Bytes
}
if len(m.Files) == 0 {
return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist)
}
if err := complete(dir, m.Files); err != nil {
return Manifest{}, fmt.Errorf("%s/%s: %w: %v", model, version, ErrIncomplete, err)
}
return m, nil
}
// complete says whether a version's files are the whole of it: nothing still
// downloading, every shard a safetensors index maps a tensor to, and every
// part of a split ("-00002-of-00004").
//
// From what the files say of each other rather than from a list written when
// the fetch began, so a version filled by hand is held to the same.
func complete(dir string, files []ManifestFile) error {
if parts, _ := filepath.Glob(filepath.Join(dir, "*"+partSuffix)); len(parts) > 0 {
var names []string
for _, p := range parts {
if b := filepath.Base(p); !strings.HasPrefix(b, ".") {
names = append(names, strings.TrimSuffix(b, partSuffix))
}
}
if len(names) > 0 {
return fmt.Errorf("still downloading %s", strings.Join(names, ", "))
}
}
have := map[string]bool{}
for _, f := range files {
have[f.Name] = true
}
var missing []string
want := map[string]bool{}
for _, f := range files {
if strings.HasSuffix(f.Name, ".safetensors.index.json") {
var idx struct {
WeightMap map[string]string `json:"weight_map"`
}
b, err := os.ReadFile(filepath.Join(dir, f.Name)) //nolint:gosec // a file in the mirror
if err != nil || json.Unmarshal(b, &idx) != nil {
return fmt.Errorf("%s cannot be read", f.Name)
}
for _, shard := range idx.WeightMap {
want[shard] = true
}
}
if m := splitPart.FindStringSubmatch(f.Name); m != nil {
total, _ := strconv.Atoi(m[3])
for i := 1; i <= total; i++ {
want[fmt.Sprintf("%s-%0*d-of-%s%s", m[1], len(m[2]), i, m[3], m[4])] = true
}
}
}
for name := range want {
if !have[name] {
missing = append(missing, name)
}
}
if len(missing) > 0 {
sort.Strings(missing)
return fmt.Errorf("missing %s", strings.Join(missing, ", "))
}
return nil
}
// splitPart matches one part of a split: prefix, part, total, extension.
var splitPart = regexp.MustCompile(`^(.+)-(\d{5})-of-(\d{5})(\.[A-Za-z0-9]+)$`)
+60
View File
@@ -177,3 +177,63 @@ func TestANameThatLeavesTheMirrorIsRefused(t *testing.T) {
}
}
}
func TestAVersionIsGivenOutOnlyWhole(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "m", "v1")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
write := func(name, body string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
write("config.json", "{}")
write("model.safetensors.index.json",
`{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}`)
write("model-00001-of-00002.safetensors", "aa")
write("model-00002-of-00002.safetensors.part", "b")
if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "still downloading") {
t.Fatalf("a version with a shard downloading was given out: %v", err)
}
// The download failed and its .part went: the index still names it.
_ = os.Remove(filepath.Join(dir, "model-00002-of-00002.safetensors.part"))
if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) ||
!strings.Contains(err.Error(), "missing model-00002-of-00002.safetensors") {
t.Fatalf("a version missing a shard its index names was given out: %v", err)
}
write("model-00002-of-00002.safetensors", "bb")
m, err := ManifestFor(root, "m", "v1")
if err != nil || len(m.Files) != 4 {
t.Fatalf("the whole version: %+v, %v", m, err)
}
// A split GGUF with no index is held to its own numbering.
gg := filepath.Join(root, "g", "v1")
_ = os.MkdirAll(gg, 0o755)
_ = os.WriteFile(filepath.Join(gg, "q-00001-of-00003.gguf"), []byte("x"), 0o644)
_ = os.WriteFile(filepath.Join(gg, "q-00003-of-00003.gguf"), []byte("x"), 0o644)
if _, err := ManifestFor(root, "g", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "q-00002-of-00003.gguf") {
t.Fatalf("a split missing its second part was given out: %v", err)
}
}
func TestAMirroredFilesChecksumIsHandedOnAndHiddenFromListings(t *testing.T) {
root := t.TempDir()
e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
added, err := Add(context.Background(), root, serving(t, "weights"), e, nil)
if err != nil {
t.Fatal(err)
}
entries, _ := List(root)
if len(entries) != 1 {
t.Errorf("the checksum was listed as a file of the model: %+v", entries)
}
m, err := ManifestFor(root, "m", "v1")
if err != nil || m.Files[0].SHA256 != added.SHA256 || added.SHA256 == "" {
t.Errorf("manifest %+v, checksum %q: %v", m, added.SHA256, err)
}
}
+14
View File
@@ -60,6 +60,9 @@ type repoSet struct {
// A shard set goes as one action — a fifteen-shard model is not fifteen
// decisions, it is one.
Sources []repoSource
// Companions are the files fetched with a safetensors set that are not
// weights: config.json, the tokenizer, the shard index.
Companions []string
// Model and Version are what the fetch form starts with. Editable, because
// the name a catalog entry uses is the operator's to choose and the
// registry's idea of it is only a good guess.
@@ -778,6 +781,17 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
}
// Safetensors are not a model on their own: vLLM loads a directory,
// and reads its config and tokenizer before any weight. Fetched as
// part of the set, or the set arrives as shards nothing can load.
if strings.HasSuffix(strings.ToLower(set.Name), ".safetensors") {
for _, f := range hf.Companions(got.Files) {
rs.Sources = append(rs.Sources,
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
rs.Companions = append(rs.Companions, f.Path)
rs.Bytes += f.Bytes
}
}
d.RepoSets = append(d.RepoSets, rs)
}
p.probeKinds(r.Context(), d.RepoSets)
+34
View File
@@ -710,3 +710,37 @@ func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) {
t.Errorf("a started download should be shown where it is followed: %s", body)
}
}
func TestASafetensorsSetIsFetchedWithWhatTheEngineReadsBesideIt(t *testing.T) {
// The field: a four-shard AWQ model mirrored as its shards alone, and
// vLLM failing at start for want of config.json.
reg := &fakeRegistry{repo: hf.Repo{
ID: "Qwen/Qwen2.5-32B-Instruct-AWQ", Revision: "abc123def456",
Files: []hf.File{
{Path: "model-00001-of-00002.safetensors", Bytes: 4_000_000_000},
{Path: "model-00002-of-00002.safetensors", Bytes: 3_000_000_000},
{Path: "model.safetensors.index.json", Bytes: 90_000},
{Path: "config.json", Bytes: 900},
{Path: "tokenizer.json", Bytes: 7_000_000},
{Path: "tokenizer_config.json", Bytes: 7_000},
{Path: "README.md", Bytes: 9_000},
{Path: "modeling_custom.py", Bytes: 40_000},
{Path: "onnx/config.json", Bytes: 900},
},
}}
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
"/admin/mirror?repo=Qwen/Qwen2.5-32B-Instruct-AWQ").Body.String()
for _, want := range []string{"config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors.index.json"} {
if !strings.Contains(body, `name="file" value="`+want+`"`) {
t.Errorf("%s is not fetched with the set", want)
}
}
for _, not := range []string{"README.md", "modeling_custom.py", "onnx/config.json"} {
if strings.Contains(body, `name="file" value="`+not+`"`) {
t.Errorf("%s was swept in with the set", not)
}
}
if !strings.Contains(body, "with 4 files the engine reads beside them") {
t.Errorf("the companions are not said; got %s", body)
}
}
+3 -1
View File
@@ -467,6 +467,7 @@ version and the old weights stay where they are.</p>
{{range .RepoSets}}<tr>
<td><code>{{.Name}}</code>
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}
{{with .Companions}}<span class="opt" title="{{join .}}">with {{len .}} files the engine reads beside them</span>{{end}}
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter: it runs on a {{.Arch}} model it was made for, not on its own">LoRA adapter · {{.Arch}}</span>
{{else if eq .Kind "model"}}<span class="opt">model{{with .Arch}} · {{.}}{{end}}</span>{{end}}</td>
<td class="num">{{bytes .Bytes}}</td>
@@ -492,7 +493,8 @@ version and the old weights stay where they are.</p>
</tr>{{end}}
</table>
<p class="note">Only the weights are listed; a repository is mostly a README and
a tokenizer. Shards of one file are one row and one button — a fifteen-shard
a tokenizer. A safetensors set is fetched with the files vLLM reads beside it —
config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard
model is one decision, not fifteen. Two separate files are two rows because they
are usually alternatives: one quantisation or the other, not both.</p>
<p class="note">The source URLs are pinned to