Let the control plane ask the mirror for a model instead of pushing one at it

The mirror holds the weights, so the mirror does the downloading. A control
plane that had to pull the bytes itself and push them here would carry every
model twice over the network, through the one machine least able to spare the
bandwidth.

Two listeners, and the split is the point. Weights stay on plain HTTP where
every node can reach them: that path is eight nodes pulling forty gigabytes,
and putting a client certificate in front of it would mean re-certifying every
agent's fetcher in order to change how models are administered. Management gets
its own port, which can be given the fleet's CA — the same tlsconf the agent
stream uses, so this is a configuration rather than a new idea.

A fetch answers 202, not 200. The bytes are not here yet, and a caller that
read it as "done" would point a catalog at a file still arriving, which every
node would then copy. Progress is asked for rather than waited on, because no
client between here and the operator will hold a connection open for an hour.

A failed fetch is remembered with its reason. Forgetting it would leave the
control plane waiting on a model that is never coming, with nothing to say why.
Status falls back to reading the disk, so a restart does not report a model
that is plainly there as unknown.

Removal takes the whole version: a model split across files is useless with one
of them missing, and half a version on disk looks present and will not load. It
refuses while a download is running, because the fetch would carry on writing
and the file would reappear, looking like the delete had silently failed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 09:34:48 +02:00
1 parent b40bc481cf
commit 16ec585411
5 files changed
+841 -25

No files matched your search

+82 -23
View File
@@ -2,6 +2,7 @@ package cli
import (
"fmt"
"log/slog"
"net/http"
"os"
"strings"
@@ -11,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// newMirrorCmd manages a directory of model weights served to the fleet.
@@ -63,7 +65,7 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
source := args[0]
file := as
if file == "" {
file = lastPathElement(source)
file = mirror.LastPathElement(source)
}
if model == "" || version == "" {
return fmt.Errorf("--model and --version are required: they are the layout")
@@ -224,40 +226,106 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
}
func newMirrorServeCmd(dir *string) *cobra.Command {
var listen string
var listen, apiListen, base string
var tlsc tlsconf.Config
c := &cobra.Command{
Use: "serve",
Short: "Serve the mirror over HTTP",
Long: "Read-only by construction: this handler has no path that writes, so a\n" +
"mirror cannot be used to replace a model with another one that every node\n" +
"would then fetch without complaint.\n\n" +
"Range requests are supported, which is what makes an interrupted download\n" +
"on a node resume rather than start again.\n\n" +
"nginx does this too, and better if you already run it. This exists so a\n" +
"mirror is one command rather than another component to install.",
Short: "Serve the mirror, and an API to manage it",
Long: "Two listeners, on purpose.\n\n" +
"The first serves the weights themselves, read-only and unauthenticated,\n" +
"because that is what eight nodes pull forty gigabytes from and putting a\n" +
"client certificate in front of it would mean re-certifying every agent's\n" +
"fetcher to change how models are administered. Range requests work, which\n" +
"is what makes an interrupted download on a node resume rather than restart.\n\n" +
"The second is the management API the control plane calls: ask the mirror to\n" +
"fetch a model, watch it arrive, remove one that is retired. It can be given\n" +
"the fleet's CA and then only holders of a certificate from it get in.\n\n" +
"The mirror does its own downloading. A control plane that had to pull the\n" +
"bytes and push them here would carry every model twice over the network,\n" +
"through the one machine least able to spare the bandwidth.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
root, err := os.Stat(*dir)
if err != nil || !root.IsDir() {
return fmt.Errorf("%s is not a directory to serve", *dir)
}
if err := tlsc.Validate(); err != nil {
return err
}
out := cmd.OutOrStdout()
log := slog.Default()
mgr := mirror.NewManager(*dir, log)
api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/"))
// http.Dir refuses paths that escape the root, which is the whole
// traversal question answered by the standard library rather than
// by me.
srv := &http.Server{
files := &http.Server{
Addr: listen,
Handler: readOnly(http.FileServer(http.Dir(*dir))),
ReadHeaderTimeout: 10 * time.Second,
}
fmt.Fprintf(cmd.OutOrStdout(), "serving %s on %s\n", *dir, listen)
return srv.ListenAndServe()
admin := &http.Server{
Addr: apiListen,
Handler: api.Handler(),
ReadHeaderTimeout: 10 * time.Second,
}
errs := make(chan error, 2)
go func() { errs <- fmt.Errorf("serving weights: %w", files.ListenAndServe()) }()
fmt.Fprintf(out, "serving %s on %s\n", *dir, listen)
go func() { errs <- fmt.Errorf("serving the API: %w", serveAPI(admin, tlsc)) }()
if tlsc.Enabled() {
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
} else {
// Said plainly rather than left to be discovered. An API that
// can delete models and start downloads, open to anyone who can
// reach the port, is a choice the operator should have made
// rather than inherited from a default.
fmt.Fprintf(out, "management API on %s, UNAUTHENTICATED — keep it on a trusted network\n", apiListen)
}
fmt.Fprintf(out, "catalog sources should point at %s\n", base)
// The first failure ends it. A mirror serving bytes but refusing
// management, or the reverse, is half a service pretending to be
// whole, and the half that works keeps the alert quiet.
return <-errs
},
}
c.Flags().StringVar(&listen, "listen", ":8081", "address to serve on")
c.Flags().StringVar(&listen, "listen", ":8081", "address to serve the weights on")
c.Flags().StringVar(&apiListen, "api-listen", ":8082", "address to serve the management API on")
c.Flags().StringVar(&base, "base", "http://localhost:8081",
"the address nodes use to reach this mirror, as it should appear in catalog sources")
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate")
c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key")
return c
}
// serveAPI starts the management listener, with mutual TLS when it is
// configured.
//
// NextProtos is cleared from what tlsconf builds. That configuration is for the
// agent stream, which needs HTTP/2 for bidirectional streams and so pins "h2";
// this is an ordinary REST API, and pinning h2 would fail the handshake for
// every HTTP/1.1 client — curl included — with an error that says nothing about
// ALPN. Empty lets net/http offer both.
func serveAPI(srv *http.Server, c tlsconf.Config) error {
if !c.Enabled() {
return srv.ListenAndServe()
}
cfg, err := c.ServerConfig()
if err != nil {
return err
}
cfg.NextProtos = nil
srv.TLSConfig = cfg
return srv.ListenAndServeTLS("", "")
}
// readOnly refuses anything but a read.
//
// http.FileServer already ignores other methods, but saying so here means the
@@ -285,12 +353,3 @@ func catalogSources(cmd *cobra.Command, server string) ([]string, error) {
}
return out, nil
}
// lastPathElement is the filename a URL ends in.
func lastPathElement(u string) string {
u = strings.SplitN(u, "?", 2)[0]
if i := strings.LastIndex(u, "/"); i >= 0 && i < len(u)-1 {
return u[i+1:]
}
return u
}
+4 -2
View File
@@ -4,6 +4,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
func TestTheMirrorRefusesEverythingButReads(t *testing.T) {
@@ -37,8 +39,8 @@ func TestTheFilenameIsTakenFromTheURLWhenNotGiven(t *testing.T) {
{"https://h/model.gguf?download=true", "model.gguf"},
{"model.gguf", "model.gguf"},
} {
if got := lastPathElement(c.url); got != c.want {
t.Errorf("lastPathElement(%q) = %q, want %q", c.url, got, c.want)
if got := mirror.LastPathElement(c.url); got != c.want {
t.Errorf("mirror.LastPathElement(%q) = %q, want %q", c.url, got, c.want)
}
}
}
+201
View File
@@ -0,0 +1,201 @@
package mirror
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"strings"
)
// API is the mirror's management interface.
//
// Management only. The weights themselves are served by an ordinary file
// handler on a separate listener, and that split is deliberate: nodes fetching
// forty gigabytes should not have to authenticate against a management API, and
// putting a client certificate in front of the byte path would mean
// re-certifying the agent's fetcher to change how models are administered.
//
// So: bytes over plain HTTP where every node can reach them, control over an
// API that can be locked down without touching them.
type API struct {
m *Manager
// base is how a catalog should refer to files here, reported alongside each
// model so the caller does not have to reconstruct the URL and get it
// subtly wrong.
base string
}
// NewAPI wraps a manager in handlers.
func NewAPI(m *Manager, base string) *API { return &API{m: m, base: base} }
// Handler routes the management endpoints.
func (a *API) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("GET /v1/models", a.list)
mux.HandleFunc("POST /v1/models", a.fetch)
mux.HandleFunc("GET /v1/models/{model}/{version}", a.status)
mux.HandleFunc("DELETE /v1/models/{model}/{version}", a.remove)
mux.HandleFunc("GET /v1/health", a.health)
return mux
}
// ModelsResponse is what the mirror holds, and where to point a catalog.
type ModelsResponse struct {
// Base is the address nodes fetch from. Returned rather than assumed
// because the caller writing a catalog entry needs the same spelling the
// mirror would use, and a mismatch there is what makes a file look unused
// when it is not.
Base string `json:"base"`
Models []Status `json:"models"`
}
func (a *API) list(w http.ResponseWriter, _ *http.Request) {
all, err := a.m.All()
if err != nil {
fail(w, http.StatusInternalServerError, err)
return
}
writeJSON(w, http.StatusOK, ModelsResponse{Base: a.base, Models: all})
}
// FetchRequest asks the mirror to download a model.
type FetchRequest struct {
Source string `json:"source"`
Model string `json:"model"`
Version string `json:"version"`
// File is optional; the last element of the source URL is used when it is
// empty, which is right almost always and wrong visibly rather than
// quietly when it is not.
File string `json:"file"`
}
// FetchResponse reports where a download stands, and where it will be.
type FetchResponse struct {
Status Status `json:"status"`
// URL is what a catalog entry should say once the download finishes. Given
// now rather than later so the operator can write the catalog while the
// bytes are still moving.
URL string `json:"url"`
}
func (a *API) fetch(w http.ResponseWriter, r *http.Request) {
var req FetchRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil {
fail(w, http.StatusBadRequest, fmt.Errorf("reading the request: %w", err))
return
}
if req.File == "" {
req.File = LastPathElement(req.Source)
}
if req.Model == "" || req.Version == "" || req.File == "" {
fail(w, http.StatusBadRequest,
errors.New("model, version and a filename are all required; the filename can come from the source URL"))
return
}
e := Entry{Model: req.Model, Version: req.Version, File: req.File}
st, err := a.m.Start(req.Source, e)
switch {
case errors.Is(err, ErrAlreadyFetching):
// Not an error the caller needs to handle: a control plane that asks
// twice for the same model wanted it once. Reporting the download
// already running says the same thing as starting it would have.
writeJSON(w, http.StatusAccepted, FetchResponse{Status: st, URL: e.URL(a.base)})
return
case err != nil:
fail(w, http.StatusBadRequest, err)
return
}
// 202, not 200: the bytes are not here yet. A caller that treats this as
// "done" would point a catalog at a file that is still arriving, and every
// node would then fetch a partial one.
writeJSON(w, http.StatusAccepted, FetchResponse{Status: st, URL: e.URL(a.base)})
}
func (a *API) status(w http.ResponseWriter, r *http.Request) {
model, version := r.PathValue("model"), r.PathValue("version")
st, ok := a.m.Status(model, version)
if !ok {
fail(w, http.StatusNotFound,
fmt.Errorf("the mirror holds nothing for %s/%s", model, version))
return
}
writeJSON(w, http.StatusOK, FetchResponse{
Status: st,
URL: Entry{Model: st.Model, Version: st.Version, File: st.File}.URL(a.base),
})
}
func (a *API) remove(w http.ResponseWriter, r *http.Request) {
model, version := r.PathValue("model"), r.PathValue("version")
if strings.ContainsAny(model+version, `/\`) {
fail(w, http.StatusBadRequest, errors.New("model and version are single names, not paths"))
return
}
// A download in flight is not deleted out from under itself: the fetch
// would carry on writing and the file would reappear, which looks like the
// delete silently failed.
if st, ok := a.m.Status(model, version); ok && st.State == Fetching {
fail(w, http.StatusConflict,
fmt.Errorf("%s/%s is still downloading; nothing was removed", model, version))
return
}
removed, err := a.m.RemoveVersion(model, version)
switch {
case errors.Is(err, os.ErrNotExist):
fail(w, http.StatusNotFound, fmt.Errorf("the mirror holds nothing for %s/%s", model, version))
return
case err != nil:
fail(w, http.StatusInternalServerError, err)
return
}
writeJSON(w, http.StatusOK, map[string]any{"removed": removed})
}
// health says the mirror is up and can read its own directory.
//
// Reading the directory rather than only answering: a mirror whose disk has
// gone away still accepts connections, and a check that only proves the process
// is running would call that healthy right up until a node tried to fetch.
func (a *API) health(w http.ResponseWriter, _ *http.Request) {
entries, err := List(a.m.root)
if err != nil {
fail(w, http.StatusServiceUnavailable, err)
return
}
var bytes int64
for _, e := range entries {
bytes += e.Bytes
}
writeJSON(w, http.StatusOK, map[string]any{
"status": "ok", "files": len(entries), "bytes": bytes, "base": a.base,
})
}
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(v)
}
// fail answers with the reason in the body.
//
// In the body because the caller is a program that will log it and a person who
// will read that log. A bare status code turns "the source URL 404s" into "the
// mirror said 400", and the next hour goes into finding out which.
func fail(w http.ResponseWriter, code int, err error) {
writeJSON(w, code, map[string]string{"error": err.Error()})
}
// LastPathElement is the filename a URL ends in.
func LastPathElement(u string) string {
u = strings.SplitN(u, "?", 2)[0]
if i := strings.LastIndex(u, "/"); i >= 0 && i < len(u)-1 {
return u[i+1:]
}
return u
}
+303
View File
@@ -0,0 +1,303 @@
package mirror
import (
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// testAPI is an API over an empty mirror, and the root it holds.
func testAPI(t *testing.T) (*API, string) {
t.Helper()
root := t.TempDir()
a := NewAPI(NewManager(root, discardLog()), "http://mirror:8081")
// A download outlives the request that started it — that is the point of
// the manager. So the temp directory must not be torn down while one is
// still writing into it. Registered after t.TempDir so it runs first.
t.Cleanup(func() {
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
all, err := a.m.All()
if err != nil {
return
}
busy := false
for _, st := range all {
busy = busy || st.State == Fetching
}
if !busy {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Error("a download was still running when the test ended")
})
return a, root
}
func call(t *testing.T, a *API, method, path, body string) (int, map[string]any) {
t.Helper()
var r io.Reader
if body != "" {
r = strings.NewReader(body)
}
req := httptest.NewRequest(method, path, r)
w := httptest.NewRecorder()
a.Handler().ServeHTTP(w, req)
var out map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
t.Fatalf("%s %s returned %d with an unreadable body %q: %v",
method, path, w.Code, w.Body.String(), err)
}
return w.Code, out
}
// waitFor polls until the condition holds, because a download runs in its own
// goroutine and the test would otherwise be asserting on a race.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatalf("timed out waiting for %s", what)
}
func TestFetchAnswersBeforeTheBytesArrive(t *testing.T) {
a, root := testAPI(t)
source := serving(t, "weights")
code, body := call(t, a, "POST", "/v1/models", `{
"source": "`+source+`", "model": "llama", "version": "2026-01-01"}`)
// 202 rather than 200: a caller that reads this as "done" points a catalog
// at a file that is still arriving.
if code != http.StatusAccepted {
t.Fatalf("expected 202 while the download runs, got %d: %v", code, body)
}
if got := body["url"]; got != "http://mirror:8081/llama/2026-01-01/model.gguf" {
t.Errorf("the response should say where the catalog will point; got %v", got)
}
waitFor(t, "the download to land on disk", func() bool {
_, err := os.Stat(filepath.Join(root, "llama", "2026-01-01", "model.gguf"))
return err == nil
})
}
func TestTheFilenameComesFromTheSourceWhenNotGiven(t *testing.T) {
a, _ := testAPI(t)
_, body := call(t, a, "POST", "/v1/models",
`{"source": "`+serving(t, "w")+`", "model": "m", "version": "v"}`)
if got := body["url"]; !strings.HasSuffix(got.(string), "/m/v/model.gguf") {
t.Errorf("the filename should be taken from the source URL; got %v", got)
}
}
func TestFetchRefusesWithoutAModelAndVersion(t *testing.T) {
a, _ := testAPI(t)
code, body := call(t, a, "POST", "/v1/models", `{"source": "http://x/model.gguf"}`)
if code != http.StatusBadRequest {
t.Fatalf("expected 400 without a model and version, got %d", code)
}
// The reason belongs in the body: a bare 400 turns a typo into an hour of
// finding out which field was wrong.
if !strings.Contains(body["error"].(string), "version") {
t.Errorf("the error should name what is missing; got %q", body["error"])
}
}
func TestAskingTwiceDoesNotStartTwoDownloads(t *testing.T) {
a, _ := testAPI(t)
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
<-release
_, _ = w.Write([]byte("weights"))
}))
defer srv.Close()
defer close(release)
body := `{"source": "` + srv.URL + `/model.gguf", "model": "m", "version": "v"}`
if code, _ := call(t, a, "POST", "/v1/models", body); code != http.StatusAccepted {
t.Fatalf("the first fetch should be accepted, got %d", code)
}
waitFor(t, "the first download to register", func() bool {
st, ok := a.m.Status("m", "v")
return ok && st.State == Fetching
})
// The second must join the first rather than open a rival writer on the
// same .part file.
code, second := call(t, a, "POST", "/v1/models", body)
if code != http.StatusAccepted {
t.Fatalf("asking again should report the running download, got %d: %v", code, second)
}
st := second["status"].(map[string]any)
if st["state"] != string(Fetching) {
t.Errorf("the second answer should describe the download already running; got %v", st["state"])
}
}
func TestAFailedFetchIsRememberedWithItsReason(t *testing.T) {
a, _ := testAPI(t)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "gone", http.StatusNotFound)
}))
defer srv.Close()
call(t, a, "POST", "/v1/models",
`{"source": "`+srv.URL+`/model.gguf", "model": "m", "version": "v"}`)
// Remembered rather than forgotten: a control plane waiting on a model that
// is never coming must be able to find out why.
waitFor(t, "the failure to be recorded", func() bool {
st, ok := a.m.Status("m", "v")
return ok && st.State == Failed
})
code, body := call(t, a, "GET", "/v1/models/m/v", "")
if code != http.StatusOK {
t.Fatalf("a failed fetch should still be reportable, got %d", code)
}
st := body["status"].(map[string]any)
if !strings.Contains(st["error"].(string), "404") {
t.Errorf("the recorded error should say what upstream answered; got %q", st["error"])
}
}
func TestStatusReadsTheDiskForModelsThisProcessDidNotFetch(t *testing.T) {
a, root := testAPI(t)
// A restart forgets every download. A model that is present must not be
// reported as unknown merely because this process did not put it there.
writeFile(t, filepath.Join(root, "m", "v", "model.gguf"), "weights")
code, body := call(t, a, "GET", "/v1/models/m/v", "")
if code != http.StatusOK {
t.Fatalf("a model on disk should be found, got %d: %v", code, body)
}
if st := body["status"].(map[string]any); st["state"] != string(Present) {
t.Errorf("expected present, got %v", st["state"])
}
}
func TestAnUnknownModelIsNotFound(t *testing.T) {
a, _ := testAPI(t)
if code, _ := call(t, a, "GET", "/v1/models/nope/v1", ""); code != http.StatusNotFound {
t.Errorf("expected 404 for a model the mirror does not hold, got %d", code)
}
}
func TestListReportsTheBaseSoCatalogsAgree(t *testing.T) {
a, root := testAPI(t)
writeFile(t, filepath.Join(root, "m", "v", "model.gguf"), "weights")
code, body := call(t, a, "GET", "/v1/models", "")
if code != http.StatusOK {
t.Fatalf("expected 200, got %d", code)
}
// Returned rather than assumed: a caller reconstructing the URL with a
// different spelling is what makes a file look unused when it is not.
if body["base"] != "http://mirror:8081" {
t.Errorf("the listing should say where nodes fetch from; got %v", body["base"])
}
if n := len(body["models"].([]any)); n != 1 {
t.Errorf("expected the one model on disk, got %d", n)
}
}
func TestRemoveTakesEveryFileOfAVersion(t *testing.T) {
a, root := testAPI(t)
// A model split across files is useless with one of them missing, so the
// version is the unit of removal, not the file.
writeFile(t, filepath.Join(root, "m", "v", "part-1.safetensors"), "a")
writeFile(t, filepath.Join(root, "m", "v", "part-2.safetensors"), "b")
code, body := call(t, a, "DELETE", "/v1/models/m/v", "")
if code != http.StatusOK {
t.Fatalf("expected 200, got %d: %v", code, body)
}
if n := len(body["removed"].([]any)); n != 2 {
t.Errorf("both files should have gone; removed %v", body["removed"])
}
if _, err := os.Stat(filepath.Join(root, "m")); !os.IsNotExist(err) {
t.Error("the emptied model directory should have gone with it")
}
}
func TestRemoveRefusesWhileADownloadIsRunning(t *testing.T) {
a, _ := testAPI(t)
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
<-release
_, _ = w.Write([]byte("weights"))
}))
defer srv.Close()
defer close(release)
call(t, a, "POST", "/v1/models",
`{"source": "`+srv.URL+`/model.gguf", "model": "m", "version": "v"}`)
waitFor(t, "the download to register", func() bool {
st, ok := a.m.Status("m", "v")
return ok && st.State == Fetching
})
// Deleting under a running fetch lets the file reappear when the download
// finishes, which looks like the delete silently failed.
code, body := call(t, a, "DELETE", "/v1/models/m/v", "")
if code != http.StatusConflict {
t.Fatalf("expected 409 while downloading, got %d: %v", code, body)
}
}
func TestRemovingSomethingAbsentIsNotFound(t *testing.T) {
a, _ := testAPI(t)
if code, _ := call(t, a, "DELETE", "/v1/models/m/v", ""); code != http.StatusNotFound {
t.Errorf("expected 404, got %d", code)
}
}
func TestHealthFailsWhenTheMirrorDirectoryIsGone(t *testing.T) {
a, root := testAPI(t)
if code, _ := call(t, a, "GET", "/v1/health", ""); code != http.StatusOK {
t.Fatalf("a readable mirror should be healthy, got %d", code)
}
// A mirror whose disk has gone away still accepts connections. A check that
// only proved the process was running would call that healthy right up
// until a node tried to fetch.
if err := os.RemoveAll(root); err != nil {
t.Fatal(err)
}
if code, _ := call(t, a, "GET", "/v1/health", ""); code != http.StatusServiceUnavailable {
t.Errorf("expected 503 once the directory is unreadable, got %d", code)
}
}
func writeFile(t *testing.T, path, body string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}
// discardLog writes nothing, so a test that provokes a failure on purpose does
// not print an error line and make a passing run look broken.
func discardLog() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
+251
View File
@@ -0,0 +1,251 @@
package mirror
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"sync"
"time"
)
// State is where a model stands in the mirror.
type State string
// The states a model can be in.
const (
// Fetching means a download is running. It is the state that makes this
// service worth an API: without it the control plane cannot tell a model
// that is arriving from one that will never arrive.
Fetching State = "fetching"
Present State = "present"
Failed State = "failed"
)
// Status is one model's state, and how far a download has got.
type Status struct {
Model string `json:"model"`
Version string `json:"version"`
File string `json:"file"`
State State `json:"state"`
Source string `json:"source,omitempty"`
BytesDone int64 `json:"bytes_done"`
// BytesTotal is 0 when the upstream server did not say. A percentage would
// then be invented, and an invented one that stops moving is worse than a
// byte count that keeps rising.
BytesTotal int64 `json:"bytes_total,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Err string `json:"error,omitempty"`
StartedAt time.Time `json:"started_at,omitzero"`
FinishedAt time.Time `json:"finished_at,omitzero"`
}
// Percent reports progress, and false when the total is unknown.
func (s Status) Percent() (int, bool) {
if s.BytesTotal <= 0 {
return 0, false
}
return int(s.BytesDone * 100 / s.BytesTotal), true
}
// Manager runs downloads in the background and remembers how they went.
//
// Background because a model is tens of gigabytes: a request that waited for
// one would time out in every client between here and the operator, and the
// operator would learn nothing about why. So a fetch is started, and its
// progress is asked for.
type Manager struct {
root string
log *slog.Logger
mu sync.Mutex
// running is keyed by model/version, which is how a second request for a
// download already in flight joins it rather than starting a rival one
// writing to the same file.
running map[string]*Status
}
// NewManager prepares a manager over a mirror root.
func NewManager(root string, log *slog.Logger) *Manager {
if log == nil {
log = slog.Default()
}
return &Manager{root: root, log: log, running: map[string]*Status{}}
}
// ErrAlreadyFetching means a download for that model and version is running.
var ErrAlreadyFetching = errors.New("a download for that version is already running")
// Start begins a download and returns once it is under way.
//
// The context bounds the caller's request, not the download: a fetch outlives
// the API call that asked for it, which is the point of doing it here rather
// than making the caller hold a connection open for an hour.
func (m *Manager) Start(source string, e Entry) (Status, error) {
if source == "" {
return Status{}, fmt.Errorf("a source URL is required")
}
key := e.Model + "/" + e.Version
m.mu.Lock()
if st, ok := m.running[key]; ok && st.State == Fetching {
// Copied under the lock. The download goroutine writes progress into
// this struct, so dereferencing it after unlocking would be a read
// racing a write — of a byte count nobody would notice was wrong.
out := *st
m.mu.Unlock()
return out, ErrAlreadyFetching
}
st := &Status{
Model: e.Model, Version: e.Version, File: e.File,
State: Fetching, Source: source, StartedAt: time.Now(),
}
m.running[key] = st
out := *st
m.mu.Unlock()
go m.run(source, e, key)
return out, nil
}
func (m *Manager) run(source string, e Entry, key string) {
// Its own context, deliberately unbounded by any request. A download is
// minutes to hours and the thing that asked for it is long gone.
ctx := context.Background()
added, err := Add(ctx, m.root, source, e, func(done, total int64) {
m.mu.Lock()
if st, ok := m.running[key]; ok {
st.BytesDone, st.BytesTotal = done, total
}
m.mu.Unlock()
})
m.mu.Lock()
defer m.mu.Unlock()
st, ok := m.running[key]
if !ok {
return
}
st.FinishedAt = time.Now()
if err != nil {
st.State, st.Err = Failed, err.Error()
// Kept rather than forgotten: a failed fetch that vanished would leave
// the control plane waiting for a model that is never coming, with
// nothing to say why.
m.log.Error("mirror fetch failed", "model", e.Model, "version", e.Version,
"source", source, "err", err)
return
}
st.State = Present
st.BytesDone, st.BytesTotal = added.Entry.Bytes, added.Entry.Bytes
st.SHA256 = added.SHA256
m.log.Info("mirror fetched a model", "model", e.Model, "version", e.Version,
"bytes", added.Entry.Bytes, "sha256", added.SHA256)
}
// Status reports one model's state, reading the disk when no download for it
// is remembered.
//
// The disk is the truth: a restart forgets every download, and a model that is
// present should not be reported as unknown merely because this process did not
// fetch it.
func (m *Manager) Status(model, version string) (Status, bool) {
key := model + "/" + version
m.mu.Lock()
st, ok := m.running[key]
if ok {
out := *st
m.mu.Unlock()
return out, true
}
m.mu.Unlock()
entries, err := List(m.root)
if err != nil {
return Status{}, false
}
for _, e := range entries {
if e.Model == model && e.Version == version {
return Status{
Model: e.Model, Version: e.Version, File: e.File,
State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes,
}, true
}
}
return Status{}, false
}
// All reports everything the mirror holds or is fetching.
//
// Both, because a control plane deciding whether to place a model needs to
// know the difference between a file that is here and one that is on its way —
// and to see that the one it asked for is on its way at all.
func (m *Manager) All() ([]Status, error) {
entries, err := List(m.root)
if err != nil {
return nil, err
}
out := make([]Status, 0, len(entries))
seen := map[string]struct{}{}
for _, e := range entries {
out = append(out, Status{
Model: e.Model, Version: e.Version, File: e.File,
State: Present, BytesDone: e.Bytes, BytesTotal: e.Bytes,
})
seen[e.Model+"/"+e.Version] = struct{}{}
}
m.mu.Lock()
for key, st := range m.running {
if _, onDisk := seen[key]; onDisk {
continue
}
out = append(out, *st)
}
m.mu.Unlock()
return out, nil
}
// Forget drops a remembered download, so a failed one can be retried.
//
// Without it a fetch that failed once would refuse every later attempt at the
// same version, and the only way through would be restarting the mirror.
func (m *Manager) Forget(model, version string) {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.running, model+"/"+version)
}
// RemoveVersion deletes every file under one model version and reports what
// went.
//
// The version is the unit, not the file: a model split across several files is
// useless with one of them missing, so removing part of it leaves a directory
// that looks present and cannot be loaded.
func (m *Manager) RemoveVersion(model, version string) ([]string, error) {
entries, err := List(m.root)
if err != nil {
return nil, err
}
var removed []string
for _, e := range entries {
if e.Model != model || e.Version != version {
continue
}
if err := Remove(m.root, e); err != nil {
return removed, err
}
removed = append(removed, e.File)
}
if len(removed) == 0 {
return nil, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist)
}
m.Forget(model, version)
m.log.Info("mirror removed a version", "model", model, "version", version, "files", len(removed))
return removed, nil
}