Let the control plane ask the mirror for a model instead of pushing one at it

The mirror holds the weights, so the mirror does the downloading. A control
plane that had to pull the bytes itself and push them here would carry every
model twice over the network, through the one machine least able to spare the
bandwidth.

Two listeners, and the split is the point. Weights stay on plain HTTP where
every node can reach them: that path is eight nodes pulling forty gigabytes,
and putting a client certificate in front of it would mean re-certifying every
agent's fetcher in order to change how models are administered. Management gets
its own port, which can be given the fleet's CA — the same tlsconf the agent
stream uses, so this is a configuration rather than a new idea.

A fetch answers 202, not 200. The bytes are not here yet, and a caller that
read it as "done" would point a catalog at a file still arriving, which every
node would then copy. Progress is asked for rather than waited on, because no
client between here and the operator will hold a connection open for an hour.

A failed fetch is remembered with its reason. Forgetting it would leave the
control plane waiting on a model that is never coming, with nothing to say why.
Status falls back to reading the disk, so a restart does not report a model
that is plainly there as unknown.

Removal takes the whole version: a model split across files is useless with one
of them missing, and half a version on disk looks present and will not load. It
refuses while a download is running, because the fetch would carry on writing
and the file would reappear, looking like the delete had silently failed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 09:34:48 +02:00
1 parent b40bc481cf
commit 16ec585411
5 files changed
+841 -25

No files matched your search

+82 -23
View File
@@ -2,6 +2,7 @@ package cli
import (
"fmt"
"log/slog"
"net/http"
"os"
"strings"
@@ -11,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// newMirrorCmd manages a directory of model weights served to the fleet.
@@ -63,7 +65,7 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
source := args[0]
file := as
if file == "" {
file = lastPathElement(source)
file = mirror.LastPathElement(source)
}
if model == "" || version == "" {
return fmt.Errorf("--model and --version are required: they are the layout")
@@ -224,40 +226,106 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
}
func newMirrorServeCmd(dir *string) *cobra.Command {
var listen string
var listen, apiListen, base string
var tlsc tlsconf.Config
c := &cobra.Command{
Use: "serve",
Short: "Serve the mirror over HTTP",
Long: "Read-only by construction: this handler has no path that writes, so a\n" +
"mirror cannot be used to replace a model with another one that every node\n" +
"would then fetch without complaint.\n\n" +
"Range requests are supported, which is what makes an interrupted download\n" +
"on a node resume rather than start again.\n\n" +
"nginx does this too, and better if you already run it. This exists so a\n" +
"mirror is one command rather than another component to install.",
Short: "Serve the mirror, and an API to manage it",
Long: "Two listeners, on purpose.\n\n" +
"The first serves the weights themselves, read-only and unauthenticated,\n" +
"because that is what eight nodes pull forty gigabytes from and putting a\n" +
"client certificate in front of it would mean re-certifying every agent's\n" +
"fetcher to change how models are administered. Range requests work, which\n" +
"is what makes an interrupted download on a node resume rather than restart.\n\n" +
"The second is the management API the control plane calls: ask the mirror to\n" +
"fetch a model, watch it arrive, remove one that is retired. It can be given\n" +
"the fleet's CA and then only holders of a certificate from it get in.\n\n" +
"The mirror does its own downloading. A control plane that had to pull the\n" +
"bytes and push them here would carry every model twice over the network,\n" +
"through the one machine least able to spare the bandwidth.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
root, err := os.Stat(*dir)
if err != nil || !root.IsDir() {
return fmt.Errorf("%s is not a directory to serve", *dir)
}
if err := tlsc.Validate(); err != nil {
return err
}
out := cmd.OutOrStdout()
log := slog.Default()
mgr := mirror.NewManager(*dir, log)
api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/"))
// http.Dir refuses paths that escape the root, which is the whole
// traversal question answered by the standard library rather than
// by me.
srv := &http.Server{
files := &http.Server{
Addr: listen,
Handler: readOnly(http.FileServer(http.Dir(*dir))),
ReadHeaderTimeout: 10 * time.Second,
}
fmt.Fprintf(cmd.OutOrStdout(), "serving %s on %s\n", *dir, listen)
return srv.ListenAndServe()
admin := &http.Server{
Addr: apiListen,
Handler: api.Handler(),
ReadHeaderTimeout: 10 * time.Second,
}
errs := make(chan error, 2)
go func() { errs <- fmt.Errorf("serving weights: %w", files.ListenAndServe()) }()
fmt.Fprintf(out, "serving %s on %s\n", *dir, listen)
go func() { errs <- fmt.Errorf("serving the API: %w", serveAPI(admin, tlsc)) }()
if tlsc.Enabled() {
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
} else {
// Said plainly rather than left to be discovered. An API that
// can delete models and start downloads, open to anyone who can
// reach the port, is a choice the operator should have made
// rather than inherited from a default.
fmt.Fprintf(out, "management API on %s, UNAUTHENTICATED — keep it on a trusted network\n", apiListen)
}
fmt.Fprintf(out, "catalog sources should point at %s\n", base)
// The first failure ends it. A mirror serving bytes but refusing
// management, or the reverse, is half a service pretending to be
// whole, and the half that works keeps the alert quiet.
return <-errs
},
}
c.Flags().StringVar(&listen, "listen", ":8081", "address to serve on")
c.Flags().StringVar(&listen, "listen", ":8081", "address to serve the weights on")
c.Flags().StringVar(&apiListen, "api-listen", ":8082", "address to serve the management API on")
c.Flags().StringVar(&base, "base", "http://localhost:8081",
"the address nodes use to reach this mirror, as it should appear in catalog sources")
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate")
c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key")
return c
}
// serveAPI starts the management listener, with mutual TLS when it is
// configured.
//
// NextProtos is cleared from what tlsconf builds. That configuration is for the
// agent stream, which needs HTTP/2 for bidirectional streams and so pins "h2";
// this is an ordinary REST API, and pinning h2 would fail the handshake for
// every HTTP/1.1 client — curl included — with an error that says nothing about
// ALPN. Empty lets net/http offer both.
func serveAPI(srv *http.Server, c tlsconf.Config) error {
if !c.Enabled() {
return srv.ListenAndServe()
}
cfg, err := c.ServerConfig()
if err != nil {
return err
}
cfg.NextProtos = nil
srv.TLSConfig = cfg
return srv.ListenAndServeTLS("", "")
}
// readOnly refuses anything but a read.
//
// http.FileServer already ignores other methods, but saying so here means the
@@ -285,12 +353,3 @@ func catalogSources(cmd *cobra.Command, server string) ([]string, error) {
}
return out, nil
}
// lastPathElement is the filename a URL ends in.
func lastPathElement(u string) string {
u = strings.SplitN(u, "?", 2)[0]
if i := strings.LastIndex(u, "/"); i >= 0 && i < len(u)-1 {
return u[i+1:]
}
return u
}
+4 -2
View File
@@ -4,6 +4,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
func TestTheMirrorRefusesEverythingButReads(t *testing.T) {
@@ -37,8 +39,8 @@ func TestTheFilenameIsTakenFromTheURLWhenNotGiven(t *testing.T) {
{"https://h/model.gguf?download=true", "model.gguf"},
{"model.gguf", "model.gguf"},
} {
if got := lastPathElement(c.url); got != c.want {
t.Errorf("lastPathElement(%q) = %q, want %q", c.url, got, c.want)
if got := mirror.LastPathElement(c.url); got != c.want {
t.Errorf("mirror.LastPathElement(%q) = %q, want %q", c.url, got, c.want)
}
}
}