Manage the model mirror with a command instead of a runbook
CI / test (push) Successful in 1m36s

The mirror was a directory and a discipline written down in a README. This
makes the three operations that matter into commands, on the mirror host,
where `llmgw keys` already sets the precedent of operator tooling that works
on files rather than through the API.

The gateway still never writes to the mirror. That property was chosen
deliberately — a mirror writable by the thing reading from it is a way to
replace a model with one every node then fetches without complaint — and a
person running a command does not take it away. `serve` states it rather than
inheriting it, so a handler added later cannot quietly make the mirror
writable.

`add` downloads to a temporary name and renames, for the reason the agent's
fetcher does: a mirror is copied to every node, so a partial file here is a
partial file everywhere. It verifies the length the server promised, reports a
checksum — the mirror is the one place a corrupt file reaches every node
rather than being caught once — and refuses to replace an existing version,
which is the whole point of the layout.

`unused` refuses to answer when it cannot read the catalog. An empty catalog
and an unreachable one produce the same list, everything, and the obvious next
step after that list is deletion. It compares by path rather than by spelling,
because a base typed with a different scheme or a trailing slash would
otherwise report a file in use as unused.

`remove` checks the catalog first and says to retire the entry before deleting
the file, because a node that has to refetch a deleted file cannot, and that
failure arrives later, on the node, looking like something else.

Verified end to end: Range requests answer 206, which is what makes an
interrupted fetch on a node resume; writes answer 405; a traversal attempt
answers 404.

Note: internal/agent/supervisor's TestSidecarRunsBeforeEngine fails under full
suite load on main as well as here — it asserts that one shell's echo lands
before another's, which the supervisor does not promise. Everything else
passes. Left alone rather than fixed in a commit about mirrors.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 09:03:07 +02:00
1 parent e86457cf02
commit b40bc481cf
9 files changed
+1059

No files matched your search

+158
View File
@@ -0,0 +1,158 @@
package mirror
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
// Added is what a download produced.
type Added struct {
Entry Entry
SHA256 string
}
// Add downloads a source into the mirror at model/version/file.
//
// Written to a temporary name and renamed when complete, for the same reason
// the agent's fetcher does it: a mirror is copied to every node, so a partial
// file here is a partial file everywhere, and a rename is the only step that
// is atomic. An interrupted download leaves a `.part` that List ignores rather
// than something a catalog could be pointed at.
//
// It refuses to replace an existing version. That refusal is the whole point of
// the layout — a version that can be overwritten takes away the ability to
// change the catalog back and still have the old weights.
func Add(ctx context.Context, root, source string, e Entry, progress func(done, total int64)) (Added, error) {
if e.Model == "" || e.Version == "" || e.File == "" {
return Added{}, fmt.Errorf("a model, a version and a filename are all required")
}
if strings.ContainsAny(e.Model+e.Version+e.File, `/\`) {
// Each is one path element. A slash would put the file somewhere the
// layout does not describe, and List would not find it again.
return Added{}, fmt.Errorf("model, version and filename are single names, not paths")
}
final := e.Path(root)
if _, err := os.Stat(final); err == nil {
return Added{}, fmt.Errorf("%s/%s/%s: %w", e.Model, e.Version, e.File, ErrExists)
}
if err := os.MkdirAll(filepath.Dir(final), 0o755); err != nil {
return Added{}, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
if err != nil {
return Added{}, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return Added{}, fmt.Errorf("fetching %s: %w", source, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return Added{}, fmt.Errorf("%s answered %s", source, resp.Status)
}
part := final + partSuffix
f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // inside the mirror root
if err != nil {
return Added{}, err
}
written, err := copyWithProgress(f, resp.Body, resp.ContentLength, progress)
closeErr := f.Close()
if err != nil {
_ = os.Remove(part)
return Added{}, err
}
if closeErr != nil {
_ = os.Remove(part)
return Added{}, closeErr
}
// A truncated download that the server ended cleanly would otherwise be
// renamed into place and copied to every node.
if resp.ContentLength > 0 && written != resp.ContentLength {
_ = os.Remove(part)
return Added{}, fmt.Errorf("%s: got %d bytes of %d", source, written, resp.ContentLength)
}
sum, err := sha256File(part)
if err != nil {
_ = os.Remove(part)
return Added{}, err
}
if err := os.Rename(part, final); err != nil {
return Added{}, err
}
e.Bytes = written
return Added{Entry: e, SHA256: sum}, nil
}
// copyWithProgress copies and reports occasionally.
//
// Occasionally rather than continuously: a forty gigabyte download is a
// hundred thousand chunks, and a line each would bury whatever else the
// operator is watching.
func copyWithProgress(dst io.Writer, src io.Reader, total int64, report func(done, total int64)) (int64, error) {
buf := make([]byte, 1<<20)
var done int64
last := time.Now()
for {
n, err := src.Read(buf)
if n > 0 {
w, werr := dst.Write(buf[:n])
done += int64(w)
if werr != nil {
return done, werr
}
if report != nil && time.Since(last) > 2*time.Second {
report(done, total)
last = time.Now()
}
}
if err == io.EOF {
if report != nil {
report(done, total)
}
return done, nil
}
if err != nil {
return done, err
}
}
}
// Remove deletes one version, and the model directory if that was the last of
// it.
//
// Leaving an empty directory behind would make `list` show a model that is not
// there, which is the kind of small lie that costs somebody an afternoon.
func Remove(root string, e Entry) error {
path := e.Path(root)
if err := os.Remove(path); err != nil {
return err
}
versionDir := filepath.Dir(path)
if empty, _ := isEmpty(versionDir); empty {
_ = os.Remove(versionDir)
}
modelDir := filepath.Dir(versionDir)
if empty, _ := isEmpty(modelDir); empty {
_ = os.Remove(modelDir)
}
return nil
}
func isEmpty(dir string) (bool, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return false, err
}
return len(entries) == 0, nil
}
+174
View File
@@ -0,0 +1,174 @@
// Package mirror manages a directory of model weights served over HTTP.
//
// The mirror is the cheapest useful thing to build around llmgw: one machine
// holding the model files so nodes fetch over your own network instead of the
// internet. Eight nodes each pulling forty gigabytes from outside is a day's
// work and on many corporate networks is not possible at all.
//
// This package is operator tooling, not part of the gateway. Nothing in the
// serving path writes here — `llmgw mirror` is run by a person, the way
// `llmgw keys` is, and the running service never has write access to the
// mirror. That is deliberate: a mirror that can be written to by the thing
// reading from it is a way to replace a model with another one that every
// node would then fetch without complaint.
package mirror
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
// Entry is one file in the mirror.
type Entry struct {
// Model and Version are the directory layout, and the layout is the point:
// a version is never overwritten, only added beside. Rolling forward is a
// new directory and one line changed in the catalog; rolling back is
// changing the line back, because the old file is still there.
Model string
Version string
File string
Bytes int64
}
// Path is where this entry lives under a root.
func (e Entry) Path(root string) string {
return filepath.Join(root, e.Model, e.Version, e.File)
}
// URL is how a catalog entry would refer to it.
func (e Entry) URL(base string) string {
return strings.TrimSuffix(base, "/") + "/" +
strings.Join([]string{e.Model, e.Version, e.File}, "/")
}
// List reads what the mirror holds.
//
// Only files three levels down, because that is the layout: anything else is
// either something a person left behind or a partial download, and reporting
// it as a model would invite somebody to point a catalog at it.
func List(root string) ([]Entry, error) {
var out []Entry
models, err := os.ReadDir(root)
if err != nil {
return nil, fmt.Errorf("reading the mirror at %s: %w", root, err)
}
for _, m := range models {
if !m.IsDir() {
continue
}
versions, err := os.ReadDir(filepath.Join(root, m.Name()))
if err != nil {
continue
}
for _, v := range versions {
if !v.IsDir() {
continue
}
files, err := os.ReadDir(filepath.Join(root, m.Name(), v.Name()))
if err != nil {
continue
}
for _, f := range files {
if f.IsDir() || strings.HasSuffix(f.Name(), partSuffix) {
continue
}
info, err := f.Info()
if err != nil {
continue
}
out = append(out, Entry{
Model: m.Name(), Version: v.Name(), File: f.Name(), Bytes: info.Size(),
})
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Model != out[j].Model {
return out[i].Model < out[j].Model
}
if out[i].Version != out[j].Version {
return out[i].Version < out[j].Version
}
return out[i].File < out[j].File
})
return out, nil
}
// ErrExists means the version is already there.
//
// Refusing rather than overwriting is the whole discipline. A version that can
// be replaced in place takes away the thing versioned paths are for: the
// ability to change the catalog back and have the old weights still be there.
var ErrExists = errors.New("that version is already in the mirror")
// Unused returns entries no catalog source refers to.
//
// `wanted` is every source in the catalog, and `base` is the address nodes use
// to reach this mirror — because a catalog refers to a file by URL and the
// mirror holds it by path, and the two are only the same string after the base
// is accounted for.
//
// A caller that cannot read the catalog must not call this with an empty list.
// Everything would come back unused, and the obvious next step would delete the
// entire mirror. The command that wraps this refuses rather than guessing.
func Unused(entries []Entry, wanted []string, base string) []Entry {
keep := map[string]struct{}{}
for _, w := range wanted {
keep[normalizeURL(w)] = struct{}{}
}
var out []Entry
for _, e := range entries {
if _, used := keep[normalizeURL(e.URL(base))]; used {
continue
}
out = append(out, e)
}
return out
}
// normalizeURL makes two spellings of one address compare equal.
//
// A catalog written by hand and a URL built from a path differ in the ways
// people differ: a trailing slash on the base, http against https when a proxy
// terminates TLS, a host written with and without its port. Comparing raw
// strings would report a file as unused because somebody typed the base
// differently, and the next step after "unused" is deletion.
func normalizeURL(u string) string {
u = strings.TrimSpace(u)
u = strings.TrimPrefix(u, "https://")
u = strings.TrimPrefix(u, "http://")
// The path is what identifies the file; the host is which mirror it came
// from, and a mirror comparing against itself has only one.
if i := strings.Index(u, "/"); i >= 0 {
u = u[i:]
}
return strings.TrimSuffix(u, "/")
}
const partSuffix = ".part"
// sha256File is the checksum of a file on disk.
//
// Reported after a download so an operator has something to compare against
// what upstream published. A mirror is the one place in this system where a
// corrupted file is copied to every node rather than caught once.
func sha256File(path string) (string, error) {
f, err := os.Open(path) //nolint:gosec // path is built from the mirror root
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
+164
View File
@@ -0,0 +1,164 @@
package mirror
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
func serving(t *testing.T, body string) string {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
return srv.URL + "/model.gguf"
}
func TestAddPlacesAFileInTheVersionedLayout(t *testing.T) {
root := t.TempDir()
got, err := Add(context.Background(), root, serving(t, "weights"),
Entry{Model: "qwen-14b", Version: "2026-09", File: "q4_k_m.gguf"}, nil)
if err != nil {
t.Fatal(err)
}
if got.Entry.Bytes != 7 {
t.Errorf("size = %d", got.Entry.Bytes)
}
if got.SHA256 == "" {
t.Error("no checksum reported; a mirror is the one place a corrupt file reaches every node")
}
if _, err := os.Stat(filepath.Join(root, "qwen-14b", "2026-09", "q4_k_m.gguf")); err != nil {
t.Errorf("the file is not where the layout says: %v", err)
}
}
func TestAddRefusesToReplaceAVersion(t *testing.T) {
// The whole discipline. A version that can be overwritten takes away the
// thing versioned paths are for: changing the catalog back and still
// having the old weights.
root := t.TempDir()
e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
if _, err := Add(context.Background(), root, serving(t, "first"), e, nil); err != nil {
t.Fatal(err)
}
_, err := Add(context.Background(), root, serving(t, "second"), e, nil)
if !errors.Is(err, ErrExists) {
t.Fatalf("overwriting a version returned %v", err)
}
b, _ := os.ReadFile(e.Path(root))
if string(b) != "first" {
t.Errorf("the original was replaced anyway: %q", b)
}
}
func TestATruncatedDownloadIsNotRenamedIntoPlace(t *testing.T) {
// A partial file here is a partial file on every node.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "100")
_, _ = w.Write([]byte("short"))
}))
defer srv.Close()
root := t.TempDir()
e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
if _, err := Add(context.Background(), root, srv.URL, e, nil); err == nil {
t.Fatal("a truncated download was accepted")
}
if _, err := os.Stat(e.Path(root)); !os.IsNotExist(err) {
t.Error("the truncated file was left where a catalog could point at it")
}
// And no .part left looking like a model.
got, _ := List(root)
if len(got) != 0 {
t.Errorf("list shows %+v after a failed download", got)
}
}
func TestPathElementsAreNamesNotPaths(t *testing.T) {
// A slash would put the file somewhere the layout does not describe, and
// List would never find it again.
root := t.TempDir()
_, err := Add(context.Background(), root, serving(t, "x"),
Entry{Model: "../escape", Version: "v", File: "f"}, nil)
if err == nil {
t.Error("a path was accepted as a name")
}
}
func TestListIgnoresAnythingNotInTheLayout(t *testing.T) {
root := t.TempDir()
if err := os.MkdirAll(filepath.Join(root, "m", "v1"), 0o755); err != nil {
t.Fatal(err)
}
for _, f := range []string{"real.gguf", "half.gguf.part"} {
if err := os.WriteFile(filepath.Join(root, "m", "v1", f), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
// A file somebody dropped at the top level, which is not a model.
if err := os.WriteFile(filepath.Join(root, "notes.txt"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
got, err := List(root)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].File != "real.gguf" {
t.Errorf("list = %+v", got)
}
}
func TestUnusedComparesByPathNotBySpelling(t *testing.T) {
// A catalog written by hand and a URL built from a path differ in the ways
// people differ. Reporting a file as unused because somebody typed the
// base differently would be followed by somebody deleting it.
entries := []Entry{
{Model: "m", Version: "v1", File: "a.gguf"},
{Model: "m", Version: "v2", File: "a.gguf"},
}
wanted := []string{
// A different scheme and a trailing slash on the base, both of which
// name the same file.
"https://models.internal/m/v1/a.gguf",
}
got := Unused(entries, wanted, "http://models.internal/")
if len(got) != 1 || got[0].Version != "v2" {
t.Errorf("unused = %+v, want only v2", got)
}
}
func TestRemoveTakesTheEmptyDirectoriesWithIt(t *testing.T) {
// An empty directory left behind makes list show a model that is not
// there, which is the kind of small lie that costs an afternoon.
root := t.TempDir()
e := Entry{Model: "m", Version: "v1", File: "f.gguf"}
if _, err := Add(context.Background(), root, serving(t, "x"), e, nil); err != nil {
t.Fatal(err)
}
if err := Remove(root, e); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, "m")); !os.IsNotExist(err) {
t.Error("an empty model directory was left behind")
}
}
func TestProgressIsReportedForALongDownload(t *testing.T) {
root := t.TempDir()
var calls int
_, err := Add(context.Background(), root, serving(t, strings.Repeat("x", 1<<20)),
Entry{Model: "m", Version: "v", File: "f"}, func(done, total int64) { calls++ })
if err != nil {
t.Fatal(err)
}
if calls == 0 {
t.Error("a download reported no progress at all")
}
}