Mirror a many-file model whole, with its checksums, for vLLM
The explorer fetches a safetensors set with its config and tokenizer; the mirror gives out a version's manifest only once nothing is downloading and every shard its index names is there, with each file's sha256, which nodes check; mirror add takes several URLs; a repository page is refused as a source. Refs #2 (item 9). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8957db30a2
commit
2735c5a318
13 files changed
+465
-34
No files matched your search
+54
-29
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
@@ -56,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
|
||||
var model, version, as string
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "add <url>",
|
||||
Use: "add <url>...",
|
||||
Short: "Download a model into the mirror",
|
||||
Long: "Downloads to a temporary name and renames when complete, so an\n" +
|
||||
"interrupted download never leaves something a catalog could be pointed\n" +
|
||||
"at. Reports the checksum: a mirror is the one place in this system where\n" +
|
||||
"a corrupted file is copied to every node rather than caught once.\n\n" +
|
||||
"Several URLs go into one version, as the files of one model: the shards\n" +
|
||||
"of a safetensors set with its config.json and tokenizer, for vLLM. The\n" +
|
||||
"version is offered to nodes only once it is whole — nothing still\n" +
|
||||
"downloading, every shard its index names present — and its source is\n" +
|
||||
"the version, ending in a slash.\n\n" +
|
||||
"Refuses to replace a version that is already here. Add a new one beside\n" +
|
||||
"it instead — that is what makes the old one still available when the new\n" +
|
||||
"one turns out to be worse.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
source := args[0]
|
||||
file := as
|
||||
if file == "" {
|
||||
file = mirror.LastPathElement(source)
|
||||
}
|
||||
if model == "" || version == "" {
|
||||
return fmt.Errorf("--model and --version are required: they are the layout")
|
||||
}
|
||||
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "fetching %s\n", source)
|
||||
|
||||
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n",
|
||||
fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
if as != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args))
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n",
|
||||
added.Entry.Model, added.Entry.Version, added.Entry.File,
|
||||
fileSize(added.Entry.Bytes), added.SHA256)
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n",
|
||||
added.Entry.URL("http://<this-host>:8081"))
|
||||
out := cmd.OutOrStdout()
|
||||
var last mirror.Added
|
||||
for _, source := range args {
|
||||
file := as
|
||||
if file == "" {
|
||||
file = mirror.LastPathElement(source)
|
||||
}
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
fmt.Fprintf(out, "fetching %s\n", source)
|
||||
added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n",
|
||||
fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n",
|
||||
added.Entry.Model, added.Entry.Version, added.Entry.File,
|
||||
fileSize(added.Entry.Bytes), added.SHA256)
|
||||
last = added
|
||||
}
|
||||
src := last.Entry.URL("http://<this-host>:8081")
|
||||
if len(args) > 1 {
|
||||
src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://<this-host>:8081"), "/") + "/"
|
||||
if m, err := mirror.ManifestFor(*dir, model, version); err != nil {
|
||||
fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err)
|
||||
} else {
|
||||
fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes))
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", src)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
|
||||
c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -405,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
m, err := mirror.ManifestFor(root, model, version)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, mirror.ErrIncomplete):
|
||||
// Said, not hidden behind a 404: a node asking now is told why
|
||||
// it cannot have the version yet, and tries again later.
|
||||
w.Header().Set("Retry-After", "60")
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
case err != nil:
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user