From 744a283cf1c23ff5b3bda0a92505671cf7c796d0 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 10:17:01 +0200 Subject: [PATCH] Keep the checksum that finishing the download used to take away MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All() preferred the disk entry wholesale, and a directory listing has no checksum in it. So the sha256 a fetch computed was reported while the bytes were still arriving and vanished the moment the file landed — it disappeared exactly when it became verifiable, which is the one moment it is for. It is the field an operator compares against what upstream published. A mirror is the single place in this system where a corrupted file is copied to every node instead of being caught once. Co-Authored-By: Claude Opus 5 (1M context) --- internal/mirror/api_test.go | 25 +++++++++++++++++++++++++ internal/mirror/manager.go | 31 ++++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/internal/mirror/api_test.go b/internal/mirror/api_test.go index 8daab1ef..0aceda9f 100644 --- a/internal/mirror/api_test.go +++ b/internal/mirror/api_test.go @@ -301,3 +301,28 @@ func writeFile(t *testing.T, path, body string) { func discardLog() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +func TestTheChecksumSurvivesTheDownloadFinishing(t *testing.T) { + a, _ := testAPI(t) + call(t, a, "POST", "/v1/models", + `{"source": "`+serving(t, "weights")+`", "model": "m", "version": "v"}`) + + waitFor(t, "the download to finish", func() bool { + st, ok := a.m.Status("m", "v") + return ok && st.State == Present + }) + + // The listing used to prefer the disk entry wholesale, and a directory has + // no checksum in it — so the sha256 vanished at the exact moment the file + // became verifiable. It is the one field an operator compares against what + // upstream published. + _, body := call(t, a, "GET", "/v1/models", "") + models := body["models"].([]any) + if len(models) != 1 { + t.Fatalf("expected the one model, got %v", models) + } + got := models[0].(map[string]any) + if got["sha256"] == nil || got["sha256"] == "" { + t.Errorf("the checksum should still be reported once the file is present; got %v", got) + } +} diff --git a/internal/mirror/manager.go b/internal/mirror/manager.go index 82c63aeb..10ac5e7a 100644 --- a/internal/mirror/manager.go +++ b/internal/mirror/manager.go @@ -190,24 +190,41 @@ func (m *Manager) All() ([]Status, error) { if err != nil { return nil, err } + // Taken before the disk entries are built so each one can be told what this + // process knows about it that the filesystem cannot say. + m.mu.Lock() + known := make(map[string]Status, len(m.running)) + for key, st := range m.running { + known[key] = *st + } + m.mu.Unlock() + out := make([]Status, 0, len(entries)) seen := map[string]struct{}{} for _, e := range entries { - out = append(out, Status{ + key := e.Model + "/" + e.Version + st := Status{ Model: e.Model, Version: e.Version, File: e.File, State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes, - }) - seen[e.Model+"/"+e.Version] = struct{}{} + } + // The checksum and the source are not recoverable from a directory + // listing, and the checksum is the one field worth having: it is what + // an operator compares against what upstream published. Dropping it in + // favour of the disk entry lost it at the exact moment the file became + // verifiable — the download finishing was what made it disappear. + if k, ok := known[key]; ok && k.File == e.File { + st.SHA256, st.Source = k.SHA256, k.Source + } + out = append(out, st) + seen[key] = struct{}{} } - m.mu.Lock() - for key, st := range m.running { + for key, st := range known { if _, onDisk := seen[key]; onDisk { continue } - out = append(out, *st) + out = append(out, st) } - m.mu.Unlock() return out, nil }