Keep the checksum that finishing the download used to take away
All() preferred the disk entry wholesale, and a directory listing has no checksum in it. So the sha256 a fetch computed was reported while the bytes were still arriving and vanished the moment the file landed — it disappeared exactly when it became verifiable, which is the one moment it is for. It is the field an operator compares against what upstream published. A mirror is the single place in this system where a corrupted file is copied to every node instead of being caught once. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7e6fb0059a
commit
744a283cf1
2 files changed
+49
-7
No files matched your search
@@ -301,3 +301,28 @@ func writeFile(t *testing.T, path, body string) {
|
||||
func discardLog() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
func TestTheChecksumSurvivesTheDownloadFinishing(t *testing.T) {
|
||||
a, _ := testAPI(t)
|
||||
call(t, a, "POST", "/v1/models",
|
||||
`{"source": "`+serving(t, "weights")+`", "model": "m", "version": "v"}`)
|
||||
|
||||
waitFor(t, "the download to finish", func() bool {
|
||||
st, ok := a.m.Status("m", "v")
|
||||
return ok && st.State == Present
|
||||
})
|
||||
|
||||
// The listing used to prefer the disk entry wholesale, and a directory has
|
||||
// no checksum in it — so the sha256 vanished at the exact moment the file
|
||||
// became verifiable. It is the one field an operator compares against what
|
||||
// upstream published.
|
||||
_, body := call(t, a, "GET", "/v1/models", "")
|
||||
models := body["models"].([]any)
|
||||
if len(models) != 1 {
|
||||
t.Fatalf("expected the one model, got %v", models)
|
||||
}
|
||||
got := models[0].(map[string]any)
|
||||
if got["sha256"] == nil || got["sha256"] == "" {
|
||||
t.Errorf("the checksum should still be reported once the file is present; got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -190,24 +190,41 @@ func (m *Manager) All() ([]Status, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Taken before the disk entries are built so each one can be told what this
|
||||
// process knows about it that the filesystem cannot say.
|
||||
m.mu.Lock()
|
||||
known := make(map[string]Status, len(m.running))
|
||||
for key, st := range m.running {
|
||||
known[key] = *st
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
out := make([]Status, 0, len(entries))
|
||||
seen := map[string]struct{}{}
|
||||
for _, e := range entries {
|
||||
out = append(out, Status{
|
||||
key := e.Model + "/" + e.Version
|
||||
st := Status{
|
||||
Model: e.Model, Version: e.Version, File: e.File,
|
||||
State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes,
|
||||
})
|
||||
seen[e.Model+"/"+e.Version] = struct{}{}
|
||||
}
|
||||
// The checksum and the source are not recoverable from a directory
|
||||
// listing, and the checksum is the one field worth having: it is what
|
||||
// an operator compares against what upstream published. Dropping it in
|
||||
// favour of the disk entry lost it at the exact moment the file became
|
||||
// verifiable — the download finishing was what made it disappear.
|
||||
if k, ok := known[key]; ok && k.File == e.File {
|
||||
st.SHA256, st.Source = k.SHA256, k.Source
|
||||
}
|
||||
out = append(out, st)
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
for key, st := range m.running {
|
||||
for key, st := range known {
|
||||
if _, onDisk := seen[key]; onDisk {
|
||||
continue
|
||||
}
|
||||
out = append(out, *st)
|
||||
out = append(out, st)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user