A mirror with peers records the checksums of files that had none, so they are copied too
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0033a530c7
commit
785e06ecbf
3 files changed
+66
No files matched your search
@@ -32,6 +32,8 @@ APIs change between minors without a deprecation period.
|
||||
- **`llmgw mirror add-peer` on a mirror with a machine of its own.** It set the second mirror up and then
|
||||
failed for want of the control plane's server.yaml; now it prints the `llmgw ctl config set mirror.peers`
|
||||
to run on a control plane.
|
||||
- **Models fetched before checksums were kept reach the second mirror.** A mirror kept in step copies only what it
|
||||
can check, and those files had no sha256; a mirror with peers now works theirs out and records them, once.
|
||||
|
||||
## v0.15.2 — 2026-10-29
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package mirror
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -24,6 +25,13 @@ type Peers struct {
|
||||
|
||||
// Run reads the peers until ctx ends.
|
||||
func (p *Peers) Run(ctx context.Context) {
|
||||
// Files fetched before checksums were kept have none, and a peer copies
|
||||
// only what it can check: they are given theirs, beside the sync.
|
||||
go func() {
|
||||
if n := p.M.FillChecksums(ctx); n > 0 {
|
||||
p.Log.Info("recorded the checksums of files that had none, so peer mirrors can copy them", "files", n)
|
||||
}
|
||||
}()
|
||||
t := time.NewTicker(PeerInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -122,3 +130,33 @@ func (p *Peers) SyncOnce(ctx context.Context, c *Client) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FillChecksums records the sha256 of every file the mirror holds without
|
||||
// one, and says how many it recorded.
|
||||
func (m *Manager) FillChecksums(ctx context.Context) int {
|
||||
entries, err := List(m.root)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, e := range entries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
path := filepath.Join(m.root, e.Model, e.Version, e.File)
|
||||
if readChecksum(path) != "" || m.isRunning(e.Model, e.Version, e.File) {
|
||||
continue
|
||||
}
|
||||
sum, err := sha256File(path)
|
||||
if err != nil {
|
||||
m.log.Warn("could not read a file's checksum", "file", path, "err", err)
|
||||
continue
|
||||
}
|
||||
if err := writeChecksum(path, sum); err != nil {
|
||||
m.log.Warn("could not record a file's checksum", "file", path, "err", err)
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -172,3 +172,29 @@ func TestAModelsAddressIsMovedToAMirrorThatAnswers(t *testing.T) {
|
||||
t.Errorf("none up: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A file fetched before checksums were kept is given one, and is then copied.
|
||||
func TestAFileWithoutAChecksumIsGivenOneAndThenCopied(t *testing.T) {
|
||||
a, b := newTestMirror(t), newTestMirror(t)
|
||||
path := filepath.Join(a.root, "old", "v1", "m.gguf")
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("fetched long ago"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pb := &Peers{M: b.m, Log: discardLog()}
|
||||
pb.SyncOnce(t.Context(), a.client(t))
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if b.has("old", "v1") {
|
||||
t.Fatal("copied without a checksum to check it by")
|
||||
}
|
||||
if n := a.m.FillChecksums(t.Context()); n != 1 || readChecksum(path) == "" {
|
||||
t.Fatalf("recorded %d", n)
|
||||
}
|
||||
if a.m.FillChecksums(t.Context()) != 0 {
|
||||
t.Error("recorded again")
|
||||
}
|
||||
pb.SyncOnce(t.Context(), a.client(t))
|
||||
waitPresent(t, b, "old", "v1")
|
||||
}
|
||||
Reference in new issue
Block a user