Mirror a many-file model whole, with its checksums, for vLLM
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m58s

The explorer fetches a safetensors set with its config and tokenizer;
the mirror gives out a version's manifest only once nothing is
downloading and every shard its index names is there, with each file's
sha256, which nodes check; mirror add takes several URLs; a repository
page is refused as a source.

Refs #2 (item 9).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 12:21:04 +02:00
1 parent 8957db30a2
commit 2735c5a318
13 files changed
+465 -34

No files matched your search

+43 -2
View File
@@ -2,6 +2,8 @@ package weights
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -26,8 +28,9 @@ type manifest struct {
Model string `json:"model"`
Version string `json:"version"`
Files []struct {
Name string `json:"name"`
Bytes int64 `json:"bytes"`
Name string `json:"name"`
Bytes int64 `json:"bytes"`
SHA256 string `json:"sha256"`
} `json:"files"`
Bytes int64 `json:"bytes"`
}
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
}
}
// Each file against the checksum the mirror took when it fetched it,
// before the set is called complete: a shard damaged on the way loads as
// a model that answers nonsense, not as an error.
for _, file := range m.Files {
if file.SHA256 == "" {
continue
}
target := filepath.Join(dir, file.Name)
got, err := fileSHA256(target)
if err != nil {
return "", err
}
if !strings.EqualFold(got, file.SHA256) {
_ = os.Remove(target)
return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
source, file.Name, got, file.SHA256)
}
}
if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
return "", err
}
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusConflict {
// Still downloading on the mirror, or a shard missing: said in the
// mirror's words, which name what is not there yet.
why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
}
if resp.StatusCode != http.StatusOK {
return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
}
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
}
return m, nil
}
func fileSHA256(path string) (string, error) {
f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
+39
View File
@@ -1,6 +1,8 @@
package weights
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
}
}
}
func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
good := "the shard as mirrored"
sum := sha256.Sum256([]byte(good))
served := "the shard damaged on the way"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasSuffix(r.URL.Path, "/") {
fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
len(served), hex.EncodeToString(sum[:]))
return
}
_, _ = w.Write([]byte(served))
}))
t.Cleanup(srv.Close)
f := &Fetcher{CacheDir: t.TempDir()}
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
t.Fatalf("a damaged shard was accepted: %v", err)
}
served = good
// The size differs, so it is fetched again; this time it matches.
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
t.Fatalf("the shard as mirrored: %v", err)
}
}
func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
}))
t.Cleanup(srv.Close)
f := &Fetcher{CacheDir: t.TempDir()}
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
!strings.Contains(err.Error(), "model-00002-of-00002") {
t.Fatalf("got %v", err)
}
}