diff --git a/CHANGELOG.md b/CHANGELOG.md index 86cfc55..7dec971 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ APIs change between minors without a deprecation period. - **`llmgw mirror add-peer` on a mirror with a machine of its own.** It set the second mirror up and then failed for want of the control plane's server.yaml; now it prints the `llmgw ctl config set mirror.peers` to run on a control plane. +- **Models fetched before checksums were kept reach the second mirror.** A mirror kept in step copies only what it + can check, and those files had no sha256; a mirror with peers now works theirs out and records them, once. ## v0.15.2 — 2026-10-29 diff --git a/internal/mirror/peer.go b/internal/mirror/peer.go index 53254ce..deeeac4 100644 --- a/internal/mirror/peer.go +++ b/internal/mirror/peer.go @@ -3,6 +3,7 @@ package mirror import ( "context" "log/slog" + "path/filepath" "strings" "time" ) @@ -24,6 +25,13 @@ type Peers struct { // Run reads the peers until ctx ends. func (p *Peers) Run(ctx context.Context) { + // Files fetched before checksums were kept have none, and a peer copies + // only what it can check: they are given theirs, beside the sync. + go func() { + if n := p.M.FillChecksums(ctx); n > 0 { + p.Log.Info("recorded the checksums of files that had none, so peer mirrors can copy them", "files", n) + } + }() t := time.NewTicker(PeerInterval) defer t.Stop() for { @@ -122,3 +130,33 @@ func (p *Peers) SyncOnce(ctx context.Context, c *Client) { } } } + +// FillChecksums records the sha256 of every file the mirror holds without +// one, and says how many it recorded. +func (m *Manager) FillChecksums(ctx context.Context) int { + entries, err := List(m.root) + if err != nil { + return 0 + } + n := 0 + for _, e := range entries { + if ctx.Err() != nil { + break + } + path := filepath.Join(m.root, e.Model, e.Version, e.File) + if readChecksum(path) != "" || m.isRunning(e.Model, e.Version, e.File) { + continue + } + sum, err := sha256File(path) + if err != nil { + m.log.Warn("could not read a file's checksum", "file", path, "err", err) + continue + } + if err := writeChecksum(path, sum); err != nil { + m.log.Warn("could not record a file's checksum", "file", path, "err", err) + continue + } + n++ + } + return n +} diff --git a/internal/mirror/peer_test.go b/internal/mirror/peer_test.go index 6ff5e29..cff3550 100644 --- a/internal/mirror/peer_test.go +++ b/internal/mirror/peer_test.go @@ -172,3 +172,29 @@ func TestAModelsAddressIsMovedToAMirrorThatAnswers(t *testing.T) { t.Errorf("none up: %s", got) } } + +// A file fetched before checksums were kept is given one, and is then copied. +func TestAFileWithoutAChecksumIsGivenOneAndThenCopied(t *testing.T) { + a, b := newTestMirror(t), newTestMirror(t) + path := filepath.Join(a.root, "old", "v1", "m.gguf") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("fetched long ago"), 0o644); err != nil { + t.Fatal(err) + } + pb := &Peers{M: b.m, Log: discardLog()} + pb.SyncOnce(t.Context(), a.client(t)) + time.Sleep(100 * time.Millisecond) + if b.has("old", "v1") { + t.Fatal("copied without a checksum to check it by") + } + if n := a.m.FillChecksums(t.Context()); n != 1 || readChecksum(path) == "" { + t.Fatalf("recorded %d", n) + } + if a.m.FillChecksums(t.Context()) != 0 { + t.Error("recorded again") + } + pb.SyncOnce(t.Context(), a.client(t)) + waitPresent(t, b, "old", "v1") +}