Mirror a many-file model whole, with its checksums, for vLLM
The explorer fetches a safetensors set with its config and tokenizer; the mirror gives out a version's manifest only once nothing is downloading and every shard its index names is there, with each file's sha256, which nodes check; mirror add takes several URLs; a repository page is refused as a source. Refs #2 (item 9). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8957db30a2
commit
2735c5a318
13 files changed
+465
-34
No files matched your search
@@ -47,6 +47,33 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
|
||||
node with two kinds of GPU, a shard could get a different card from the one it
|
||||
was given.
|
||||
|
||||
### A model of many files, mirrored whole for vLLM
|
||||
|
||||
vLLM loads a directory — shards, config.json, the tokenizer — and does not
|
||||
read GGUF. Mirroring one could go wrong several ways: the explorer offered the
|
||||
shards alone, a node could be handed a version still downloading or missing a
|
||||
shard, and a repository's page given as a model's source was fetched as one
|
||||
file of HTML.
|
||||
|
||||
- **The explorer fetches a safetensors set with what the engine reads beside
|
||||
it**: config.json, generation_config.json, the tokenizer files, the shard
|
||||
index, the quantisation config. Never code: a `.py` would be the engine
|
||||
running what the repository ships, which is to be chosen on purpose.
|
||||
- **A version is offered to nodes only whole.** The mirror refuses its
|
||||
manifest (409, with what is missing) while a file is still downloading, or
|
||||
while a shard its `model.safetensors.index.json` names, or a part of a
|
||||
split (`-00002-of-00004`), is absent. A node reports "not complete on the
|
||||
mirror yet" and is tried again.
|
||||
- **Checksums go with the files.** The mirror keeps each file's sha256 beside
|
||||
it and the manifest carries it; a node checks every file of a set against
|
||||
it before using the set, and fetches one that differs again.
|
||||
- **`llmgw mirror add` takes several URLs** into one version.
|
||||
- **A Hugging Face repository page is refused as a source**, saying to mirror
|
||||
it or to give the repository id for vLLM to fetch itself.
|
||||
|
||||
With a vLLM profile of `tp: 4`, placement reserves four GPUs of one node for
|
||||
such a model, as before.
|
||||
|
||||
### Signing up with an address that has an account
|
||||
|
||||
On PostgreSQL, signing up with an address already registered showed
|
||||
|
||||
@@ -2,6 +2,8 @@ package weights
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -26,8 +28,9 @@ type manifest struct {
|
||||
Model string `json:"model"`
|
||||
Version string `json:"version"`
|
||||
Files []struct {
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
} `json:"files"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
|
||||
}
|
||||
}
|
||||
|
||||
// Each file against the checksum the mirror took when it fetched it,
|
||||
// before the set is called complete: a shard damaged on the way loads as
|
||||
// a model that answers nonsense, not as an error.
|
||||
for _, file := range m.Files {
|
||||
if file.SHA256 == "" {
|
||||
continue
|
||||
}
|
||||
target := filepath.Join(dir, file.Name)
|
||||
got, err := fileSHA256(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !strings.EqualFold(got, file.SHA256) {
|
||||
_ = os.Remove(target)
|
||||
return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
|
||||
source, file.Name, got, file.SHA256)
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
|
||||
return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusConflict {
|
||||
// Still downloading on the mirror, or a shard missing: said in the
|
||||
// mirror's words, which name what is not there yet.
|
||||
why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
|
||||
}
|
||||
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
|
||||
good := "the shard as mirrored"
|
||||
sum := sha256.Sum256([]byte(good))
|
||||
served := "the shard damaged on the way"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/") {
|
||||
fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
|
||||
len(served), hex.EncodeToString(sum[:]))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(served))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
|
||||
t.Fatalf("a damaged shard was accepted: %v", err)
|
||||
}
|
||||
served = good
|
||||
// The size differs, so it is fetched again; this time it matches.
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
t.Fatalf("the shard as mirrored: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
|
||||
!strings.Contains(err.Error(), "model-00002-of-00002") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
+54
-29
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
@@ -56,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
|
||||
var model, version, as string
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "add <url>",
|
||||
Use: "add <url>...",
|
||||
Short: "Download a model into the mirror",
|
||||
Long: "Downloads to a temporary name and renames when complete, so an\n" +
|
||||
"interrupted download never leaves something a catalog could be pointed\n" +
|
||||
"at. Reports the checksum: a mirror is the one place in this system where\n" +
|
||||
"a corrupted file is copied to every node rather than caught once.\n\n" +
|
||||
"Several URLs go into one version, as the files of one model: the shards\n" +
|
||||
"of a safetensors set with its config.json and tokenizer, for vLLM. The\n" +
|
||||
"version is offered to nodes only once it is whole — nothing still\n" +
|
||||
"downloading, every shard its index names present — and its source is\n" +
|
||||
"the version, ending in a slash.\n\n" +
|
||||
"Refuses to replace a version that is already here. Add a new one beside\n" +
|
||||
"it instead — that is what makes the old one still available when the new\n" +
|
||||
"one turns out to be worse.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
source := args[0]
|
||||
file := as
|
||||
if file == "" {
|
||||
file = mirror.LastPathElement(source)
|
||||
}
|
||||
if model == "" || version == "" {
|
||||
return fmt.Errorf("--model and --version are required: they are the layout")
|
||||
}
|
||||
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "fetching %s\n", source)
|
||||
|
||||
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n",
|
||||
fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
if as != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args))
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n",
|
||||
added.Entry.Model, added.Entry.Version, added.Entry.File,
|
||||
fileSize(added.Entry.Bytes), added.SHA256)
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n",
|
||||
added.Entry.URL("http://<this-host>:8081"))
|
||||
out := cmd.OutOrStdout()
|
||||
var last mirror.Added
|
||||
for _, source := range args {
|
||||
file := as
|
||||
if file == "" {
|
||||
file = mirror.LastPathElement(source)
|
||||
}
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
fmt.Fprintf(out, "fetching %s\n", source)
|
||||
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n",
|
||||
fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n",
|
||||
added.Entry.Model, added.Entry.Version, added.Entry.File,
|
||||
fileSize(added.Entry.Bytes), added.SHA256)
|
||||
last = added
|
||||
}
|
||||
src := last.Entry.URL("http://<this-host>:8081")
|
||||
if len(args) > 1 {
|
||||
src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://<this-host>:8081"), "/") + "/"
|
||||
if m, err := mirror.ManifestFor(*dir, model, version); err != nil {
|
||||
fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", src)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
|
||||
c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -405,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
m, err := mirror.ManifestFor(root, model, version)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, mirror.ErrIncomplete):
|
||||
// Said, not hidden behind a 404: a node asking now is told why
|
||||
// it cannot have the version yet, and tries again later.
|
||||
w.Header().Set("Retry-After", "60")
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
case err != nil:
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -333,6 +333,13 @@ func (m Model) validate() error {
|
||||
if m.Source == "" {
|
||||
return fmt.Errorf("model %q: source is required", m.Name)
|
||||
}
|
||||
if repo, ok := repositoryPage(m.Source); ok {
|
||||
// A page, not weights: a node fetches the HTML as one file and the
|
||||
// engine fails at start, on a path that looks right.
|
||||
return fmt.Errorf("model %q: source %s is a Hugging Face repository page, not its files; "+
|
||||
"mirror it and use the version's source ending in /, or give %q for vLLM to fetch itself",
|
||||
m.Name, m.Source, repo)
|
||||
}
|
||||
if err := validateAdapters(m); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -719,3 +726,21 @@ func (m Model) engineAdapters() []engines.Adapter {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// repositoryPage reports whether source is a Hugging Face repository's page
|
||||
// (https://huggingface.co/org/repo, or one of its tree views) rather than a
|
||||
// file under /resolve/, and the repository's id.
|
||||
func repositoryPage(source string) (string, bool) {
|
||||
for _, host := range []string{"https://huggingface.co/", "http://huggingface.co/", "https://hf.co/"} {
|
||||
rest, ok := strings.CutPrefix(source, host)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(strings.Trim(rest, "/"), "/")
|
||||
if len(parts) < 2 || (len(parts) > 2 && parts[2] == "resolve") {
|
||||
return "", false
|
||||
}
|
||||
return parts[0] + "/" + parts[1], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -347,3 +347,22 @@ func TestALlamaCppReplicaOnSeveralGPUsSaysHowManyAsGPUsPerNode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryPageIsNotASource(t *testing.T) {
|
||||
for _, src := range []string{
|
||||
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ",
|
||||
"https://huggingface.co/Qwen/Qwen2.5-32B-Instruct-AWQ/tree/main",
|
||||
} {
|
||||
m := model("awq")
|
||||
m.Source = src
|
||||
_, err := New([]Model{m})
|
||||
if err == nil || !strings.Contains(err.Error(), `"Qwen/Qwen2.5-32B-Instruct-AWQ"`) {
|
||||
t.Errorf("%s: %v", src, err)
|
||||
}
|
||||
}
|
||||
m := model("file")
|
||||
m.Source = "https://huggingface.co/Qwen/X-GGUF/resolve/abc123/x.gguf"
|
||||
if _, err := New([]Model{m}); err != nil {
|
||||
t.Errorf("a file under /resolve/ is a source: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -258,6 +258,41 @@ func (f File) IsWeights() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// companionNames are the files beside safetensors weights that an engine
|
||||
// loading the directory reads: the model's shape, its tokenizer, how it was
|
||||
// quantised, and which shard holds which tensor.
|
||||
var companionNames = map[string]bool{
|
||||
"config.json": true, "generation_config.json": true,
|
||||
"tokenizer.json": true, "tokenizer_config.json": true, "tokenizer.model": true,
|
||||
"special_tokens_map.json": true, "added_tokens.json": true,
|
||||
"vocab.json": true, "vocab.txt": true, "merges.txt": true,
|
||||
"chat_template.jinja": true, "chat_template.json": true,
|
||||
"preprocessor_config.json": true, "processor_config.json": true,
|
||||
"quantize_config.json": true, "quant_config.json": true,
|
||||
}
|
||||
|
||||
// Companions are the files of a repository an engine reads beside a
|
||||
// safetensors set: config.json and the tokenizer, the shard index. Only
|
||||
// those at the top: the directory an engine is given is one level deep.
|
||||
//
|
||||
// Never code (a .py): a model that needs its own Python to load asks the
|
||||
// engine to run what the repository ships, which is a decision for whoever
|
||||
// runs the engine, made on purpose, not a file swept in with the tokenizer.
|
||||
func Companions(files []File) []File {
|
||||
var out []File
|
||||
for _, f := range files {
|
||||
name := f.Path
|
||||
if strings.Contains(name, "/") {
|
||||
continue
|
||||
}
|
||||
if companionNames[name] || strings.HasSuffix(name, ".safetensors.index.json") ||
|
||||
strings.HasSuffix(name, ".tiktoken") {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Repo is a repository's current revision and what is in it.
|
||||
type Repo struct {
|
||||
ID string
|
||||
|
||||
@@ -94,6 +94,13 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
|
||||
_ = os.Remove(part)
|
||||
return Added{}, err
|
||||
}
|
||||
// The checksum beside the file, before the file is in place: a manifest
|
||||
// listing the file then always has its checksum to hand on, and a node
|
||||
// can tell a file damaged on the way from the one that was mirrored.
|
||||
if err := writeChecksum(final, sum); err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Added{}, err
|
||||
}
|
||||
if err := os.Rename(part, final); err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
@@ -146,6 +153,9 @@ func Remove(root string, e Entry) error {
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
// Its checksum goes with it; left, it would keep the directory from
|
||||
// being empty, and so from going.
|
||||
_ = os.Remove(checksumPath(path))
|
||||
versionDir := filepath.Dir(path)
|
||||
if empty, _ := isEmpty(versionDir); empty {
|
||||
_ = os.Remove(versionDir)
|
||||
|
||||
+102
-2
@@ -16,12 +16,15 @@ package mirror
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -91,7 +94,7 @@ func List(root string) ([]Entry, error) {
|
||||
continue
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) {
|
||||
if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) || strings.HasPrefix(f.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
info, err := f.Info()
|
||||
@@ -169,6 +172,31 @@ func normalizeURL(u string) string {
|
||||
|
||||
const partSuffix = ".part"
|
||||
|
||||
// checksumPath is where a file's sha256 is kept: beside it, hidden, so no
|
||||
// listing takes it for a file of the model and no name a person gives a
|
||||
// file (which may not start with a dot) can be the same.
|
||||
func checksumPath(file string) string {
|
||||
return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".sha256")
|
||||
}
|
||||
|
||||
func writeChecksum(file, sum string) error {
|
||||
tmp := checksumPath(file) + partSuffix
|
||||
if err := os.WriteFile(tmp, []byte(sum+"\n"), 0o644); err != nil { //nolint:gosec // a checksum, for anyone to read
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, checksumPath(file))
|
||||
}
|
||||
|
||||
// readChecksum is a file's recorded sha256, or "" for a file put in place by
|
||||
// hand, which has none.
|
||||
func readChecksum(file string) string {
|
||||
b, err := os.ReadFile(checksumPath(file)) //nolint:gosec // beside a file in the mirror
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// sha256File is the checksum of a file on disk.
|
||||
//
|
||||
// Reported after a download so an operator has something to compare against
|
||||
@@ -205,8 +233,16 @@ type Manifest struct {
|
||||
type ManifestFile struct {
|
||||
Name string `json:"name"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
// SHA256 is the file's checksum when the mirror fetched it, for a node
|
||||
// to check what it downloaded; empty for a file put there by hand.
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
}
|
||||
|
||||
// ErrIncomplete is a version not yet whole: a download still running, or a
|
||||
// shard its own index names missing. Its manifest is not given out, so no
|
||||
// node starts a model from part of it.
|
||||
var ErrIncomplete = errors.New("this version is not complete")
|
||||
|
||||
// ManifestFor lists the files of one model version.
|
||||
//
|
||||
// Built from the directory rather than from a file written alongside it. A
|
||||
@@ -219,15 +255,79 @@ func ManifestFor(root, model, version string) (Manifest, error) {
|
||||
return Manifest{}, err
|
||||
}
|
||||
m := Manifest{Model: model, Version: version}
|
||||
dir := filepath.Join(root, model, version)
|
||||
for _, e := range entries {
|
||||
if e.Model != model || e.Version != version {
|
||||
continue
|
||||
}
|
||||
m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes})
|
||||
m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes,
|
||||
SHA256: readChecksum(filepath.Join(dir, e.File))})
|
||||
m.Bytes += e.Bytes
|
||||
}
|
||||
if len(m.Files) == 0 {
|
||||
return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist)
|
||||
}
|
||||
if err := complete(dir, m.Files); err != nil {
|
||||
return Manifest{}, fmt.Errorf("%s/%s: %w: %v", model, version, ErrIncomplete, err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// complete says whether a version's files are the whole of it: nothing still
|
||||
// downloading, every shard a safetensors index maps a tensor to, and every
|
||||
// part of a split ("-00002-of-00004").
|
||||
//
|
||||
// From what the files say of each other rather than from a list written when
|
||||
// the fetch began, so a version filled by hand is held to the same.
|
||||
func complete(dir string, files []ManifestFile) error {
|
||||
if parts, _ := filepath.Glob(filepath.Join(dir, "*"+partSuffix)); len(parts) > 0 {
|
||||
var names []string
|
||||
for _, p := range parts {
|
||||
if b := filepath.Base(p); !strings.HasPrefix(b, ".") {
|
||||
names = append(names, strings.TrimSuffix(b, partSuffix))
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return fmt.Errorf("still downloading %s", strings.Join(names, ", "))
|
||||
}
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, f := range files {
|
||||
have[f.Name] = true
|
||||
}
|
||||
var missing []string
|
||||
want := map[string]bool{}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Name, ".safetensors.index.json") {
|
||||
var idx struct {
|
||||
WeightMap map[string]string `json:"weight_map"`
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, f.Name)) //nolint:gosec // a file in the mirror
|
||||
if err != nil || json.Unmarshal(b, &idx) != nil {
|
||||
return fmt.Errorf("%s cannot be read", f.Name)
|
||||
}
|
||||
for _, shard := range idx.WeightMap {
|
||||
want[shard] = true
|
||||
}
|
||||
}
|
||||
if m := splitPart.FindStringSubmatch(f.Name); m != nil {
|
||||
total, _ := strconv.Atoi(m[3])
|
||||
for i := 1; i <= total; i++ {
|
||||
want[fmt.Sprintf("%s-%0*d-of-%s%s", m[1], len(m[2]), i, m[3], m[4])] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for name := range want {
|
||||
if !have[name] {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
sort.Strings(missing)
|
||||
return fmt.Errorf("missing %s", strings.Join(missing, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitPart matches one part of a split: prefix, part, total, extension.
|
||||
var splitPart = regexp.MustCompile(`^(.+)-(\d{5})-of-(\d{5})(\.[A-Za-z0-9]+)$`)
|
||||
@@ -177,3 +177,63 @@ func TestANameThatLeavesTheMirrorIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVersionIsGivenOutOnlyWhole(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "m", "v1")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(name, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("config.json", "{}")
|
||||
write("model.safetensors.index.json",
|
||||
`{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}`)
|
||||
write("model-00001-of-00002.safetensors", "aa")
|
||||
write("model-00002-of-00002.safetensors.part", "b")
|
||||
|
||||
if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "still downloading") {
|
||||
t.Fatalf("a version with a shard downloading was given out: %v", err)
|
||||
}
|
||||
// The download failed and its .part went: the index still names it.
|
||||
_ = os.Remove(filepath.Join(dir, "model-00002-of-00002.safetensors.part"))
|
||||
if _, err := ManifestFor(root, "m", "v1"); !errors.Is(err, ErrIncomplete) ||
|
||||
!strings.Contains(err.Error(), "missing model-00002-of-00002.safetensors") {
|
||||
t.Fatalf("a version missing a shard its index names was given out: %v", err)
|
||||
}
|
||||
write("model-00002-of-00002.safetensors", "bb")
|
||||
m, err := ManifestFor(root, "m", "v1")
|
||||
if err != nil || len(m.Files) != 4 {
|
||||
t.Fatalf("the whole version: %+v, %v", m, err)
|
||||
}
|
||||
|
||||
// A split GGUF with no index is held to its own numbering.
|
||||
gg := filepath.Join(root, "g", "v1")
|
||||
_ = os.MkdirAll(gg, 0o755)
|
||||
_ = os.WriteFile(filepath.Join(gg, "q-00001-of-00003.gguf"), []byte("x"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(gg, "q-00003-of-00003.gguf"), []byte("x"), 0o644)
|
||||
if _, err := ManifestFor(root, "g", "v1"); !errors.Is(err, ErrIncomplete) || !strings.Contains(err.Error(), "q-00002-of-00003.gguf") {
|
||||
t.Fatalf("a split missing its second part was given out: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMirroredFilesChecksumIsHandedOnAndHiddenFromListings(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
|
||||
added, err := Add(context.Background(), root, serving(t, "weights"), e, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, _ := List(root)
|
||||
if len(entries) != 1 {
|
||||
t.Errorf("the checksum was listed as a file of the model: %+v", entries)
|
||||
}
|
||||
m, err := ManifestFor(root, "m", "v1")
|
||||
if err != nil || m.Files[0].SHA256 != added.SHA256 || added.SHA256 == "" {
|
||||
t.Errorf("manifest %+v, checksum %q: %v", m, added.SHA256, err)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,9 @@ type repoSet struct {
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
// decisions, it is one.
|
||||
Sources []repoSource
|
||||
// Companions are the files fetched with a safetensors set that are not
|
||||
// weights: config.json, the tokenizer, the shard index.
|
||||
Companions []string
|
||||
// Model and Version are what the fetch form starts with. Editable, because
|
||||
// the name a catalog entry uses is the operator's to choose and the
|
||||
// registry's idea of it is only a good guess.
|
||||
@@ -778,6 +781,17 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
}
|
||||
// Safetensors are not a model on their own: vLLM loads a directory,
|
||||
// and reads its config and tokenizer before any weight. Fetched as
|
||||
// part of the set, or the set arrives as shards nothing can load.
|
||||
if strings.HasSuffix(strings.ToLower(set.Name), ".safetensors") {
|
||||
for _, f := range hf.Companions(got.Files) {
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
rs.Companions = append(rs.Companions, f.Path)
|
||||
rs.Bytes += f.Bytes
|
||||
}
|
||||
}
|
||||
d.RepoSets = append(d.RepoSets, rs)
|
||||
}
|
||||
p.probeKinds(r.Context(), d.RepoSets)
|
||||
|
||||
@@ -710,3 +710,37 @@ func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) {
|
||||
t.Errorf("a started download should be shown where it is followed: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASafetensorsSetIsFetchedWithWhatTheEngineReadsBesideIt(t *testing.T) {
|
||||
// The field: a four-shard AWQ model mirrored as its shards alone, and
|
||||
// vLLM failing at start for want of config.json.
|
||||
reg := &fakeRegistry{repo: hf.Repo{
|
||||
ID: "Qwen/Qwen2.5-32B-Instruct-AWQ", Revision: "abc123def456",
|
||||
Files: []hf.File{
|
||||
{Path: "model-00001-of-00002.safetensors", Bytes: 4_000_000_000},
|
||||
{Path: "model-00002-of-00002.safetensors", Bytes: 3_000_000_000},
|
||||
{Path: "model.safetensors.index.json", Bytes: 90_000},
|
||||
{Path: "config.json", Bytes: 900},
|
||||
{Path: "tokenizer.json", Bytes: 7_000_000},
|
||||
{Path: "tokenizer_config.json", Bytes: 7_000},
|
||||
{Path: "README.md", Bytes: 9_000},
|
||||
{Path: "modeling_custom.py", Bytes: 40_000},
|
||||
{Path: "onnx/config.json", Bytes: 900},
|
||||
},
|
||||
}}
|
||||
body := mirrorGet(t, mirrorPageWith(t, &fakeMirror{}, reg, true),
|
||||
"/admin/mirror?repo=Qwen/Qwen2.5-32B-Instruct-AWQ").Body.String()
|
||||
for _, want := range []string{"config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors.index.json"} {
|
||||
if !strings.Contains(body, `name="file" value="`+want+`"`) {
|
||||
t.Errorf("%s is not fetched with the set", want)
|
||||
}
|
||||
}
|
||||
for _, not := range []string{"README.md", "modeling_custom.py", "onnx/config.json"} {
|
||||
if strings.Contains(body, `name="file" value="`+not+`"`) {
|
||||
t.Errorf("%s was swept in with the set", not)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(body, "with 4 files the engine reads beside them") {
|
||||
t.Errorf("the companions are not said; got %s", body)
|
||||
}
|
||||
}
|
||||
@@ -467,6 +467,7 @@ version and the old weights stay where they are.</p>
|
||||
{{range .RepoSets}}<tr>
|
||||
<td><code>{{.Name}}</code>
|
||||
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}
|
||||
{{with .Companions}}<span class="opt" title="{{join .}}">with {{len .}} files the engine reads beside them</span>{{end}}
|
||||
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter: it runs on a {{.Arch}} model it was made for, not on its own">LoRA adapter · {{.Arch}}</span>
|
||||
{{else if eq .Kind "model"}}<span class="opt">model{{with .Arch}} · {{.}}{{end}}</span>{{end}}</td>
|
||||
<td class="num">{{bytes .Bytes}}</td>
|
||||
@@ -492,7 +493,8 @@ version and the old weights stay where they are.</p>
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<p class="note">Only the weights are listed; a repository is mostly a README and
|
||||
a tokenizer. Shards of one file are one row and one button — a fifteen-shard
|
||||
a tokenizer. A safetensors set is fetched with the files vLLM reads beside it —
|
||||
config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard
|
||||
model is one decision, not fifteen. Two separate files are two rows because they
|
||||
are usually alternatives: one quantisation or the other, not both.</p>
|
||||
<p class="note">The source URLs are pinned to
|
||||
|
||||
Reference in new issue
Block a user