package cli import ( "encoding/json" "fmt" "log/slog" "net/http" "os" "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // newMirrorCmd manages a directory of model weights served to the fleet. // // Operator tooling, run on the mirror host, the way `llmgw keys` is run on the // machine holding the store. The gateway never writes here: a mirror that can // be written to by the thing reading from it is a way to replace a model with // another one that every node would then fetch without complaint. func newMirrorCmd() *cobra.Command { var dir string c := &cobra.Command{ Use: "mirror", Short: "Hold model weights for the fleet to fetch from", Long: "One machine holding the model files, so nodes fetch over your own\n" + "network instead of the internet. Eight nodes each pulling forty gigabytes\n" + "from outside is a day's work, and on many corporate networks it is not\n" + "possible at all.\n\n" + "Files live at //, and a version is never\n" + "overwritten. That is what makes rolling back possible: changing the\n" + "catalog line back works because the old weights are still there.", } c.PersistentFlags().StringVar(&dir, "dir", "/srv/models", "the mirror's root directory") c.AddCommand( newMirrorAddCmd(&dir), newMirrorListCmd(&dir), newMirrorUnusedCmd(&dir), newMirrorRemoveCmd(&dir), newMirrorServeCmd(&dir), ) return c } func newMirrorAddCmd(dir *string) *cobra.Command { var model, version, as string c := &cobra.Command{ Use: "add ", Short: "Download a model into the mirror", Long: "Downloads to a temporary name and renames when complete, so an\n" + "interrupted download never leaves something a catalog could be pointed\n" + "at. Reports the checksum: a mirror is the one place in this system where\n" + "a corrupted file is copied to every node rather than caught once.\n\n" + "Refuses to replace a version that is already here. Add a new one beside\n" + "it instead — that is what makes the old one still available when the new\n" + "one turns out to be worse.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { source := args[0] file := as if file == "" { file = mirror.LastPathElement(source) } if model == "" || version == "" { return fmt.Errorf("--model and --version are required: they are the layout") } e := mirror.Entry{Model: model, Version: version, File: file} out := cmd.OutOrStdout() fmt.Fprintf(out, "fetching %s\n", source) added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) { if total > 0 { fmt.Fprintf(out, " %s of %s (%d%%)\n", fileSize(done), fileSize(total), done*100/total) return } fmt.Fprintf(out, " %s so far\n", fileSize(done)) }) if err != nil { return err } fmt.Fprintf(out, "\n%s/%s/%s %s\n sha256 %s\n", added.Entry.Model, added.Entry.Version, added.Entry.File, fileSize(added.Entry.Bytes), added.SHA256) fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", added.Entry.URL("http://:8081")) return nil }, } c.Flags().StringVar(&model, "model", "", "the model's name, one directory level") c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well") c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted") return c } func newMirrorListCmd(dir *string) *cobra.Command { return &cobra.Command{ Use: "list", Short: "Show what the mirror holds", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { entries, err := mirror.List(*dir) if err != nil { return err } if len(entries) == 0 { fmt.Fprintf(cmd.OutOrStdout(), "The mirror at %s is empty.\n", *dir) return nil } w := tab() fmt.Fprintln(w, "MODEL\tVERSION\tFILE\tSIZE") var total int64 for _, e := range entries { fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", e.Model, e.Version, e.File, fileSize(e.Bytes)) total += e.Bytes } _ = w.Flush() fmt.Fprintf(cmd.OutOrStdout(), "\n%s in %d files.\n", fileSize(total), len(entries)) return nil }, } } func newMirrorUnusedCmd(dir *string) *cobra.Command { var server, base string c := &cobra.Command{ Use: "unused", Short: "Show files no model in the catalog refers to", Long: "Compares what is here against every source in the catalog, so the\n" + "answer is what could be deleted rather than what looks old.\n\n" + "It refuses to answer when it cannot read the catalog. An empty catalog\n" + "and an unreachable one produce the same list — everything — and the\n" + "obvious next step after that list is deletion.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { entries, err := mirror.List(*dir) if err != nil { return err } sources, err := catalogSources(cmd, server) if err != nil { // Named as the reason rather than folded into a generic // failure: an operator who sees "no catalog" will fix the // address, and one who sees a list will start deleting. return fmt.Errorf("cannot read the catalog, so nothing can be called unused: %w", err) } unused := mirror.Unused(entries, sources, base) out := cmd.OutOrStdout() if len(unused) == 0 { fmt.Fprintf(out, "Every file here is referred to by a model. (%d files, %d catalog sources.)\n", len(entries), len(sources)) return nil } w := tab() fmt.Fprintln(w, "MODEL\tVERSION\tFILE\tSIZE") var total int64 for _, e := range unused { fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", e.Model, e.Version, e.File, fileSize(e.Bytes)) total += e.Bytes } _ = w.Flush() fmt.Fprintf(out, "\n%s in %d files that no model refers to.\n", fileSize(total), len(unused)) fmt.Fprintln(out, "\nBefore deleting: a node that has to refetch one of these will not be\n"+ "able to. Retire the catalog entry first, let the fleet settle, then remove.") return nil }, } c.Flags().StringVar(&server, "server", "127.0.0.1:9090", "admin API address to read the catalog from") c.Flags().StringVar(&base, "base", "http://localhost:8081", "the address nodes use to reach this mirror, as it appears in catalog sources") return c } func newMirrorRemoveCmd(dir *string) *cobra.Command { var model, version, file, server, base string var force bool c := &cobra.Command{ Use: "remove", Short: "Delete one version from the mirror", Long: "Checks the catalog first, and refuses if a model still refers to it —\n" + "a node that has to refetch a deleted file cannot, and the failure arrives\n" + "later, on the node, looking like something else.\n\n" + "Retire the catalog entry first, let the fleet settle, then remove.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { if model == "" || version == "" || file == "" { return fmt.Errorf("--model, --version and --file are all required") } e := mirror.Entry{Model: model, Version: version, File: file} if !force { sources, err := catalogSources(cmd, server) if err != nil { return fmt.Errorf("cannot read the catalog to check whether this is still in use: %w\n"+ " Pass --force to remove it anyway", err) } if len(mirror.Unused([]mirror.Entry{e}, sources, base)) == 0 { return fmt.Errorf("a model still refers to %s/%s/%s; retire the catalog entry first", model, version, file) } } if err := mirror.Remove(*dir, e); err != nil { return err } fmt.Fprintf(cmd.OutOrStdout(), "removed %s/%s/%s\n", model, version, file) return nil }, } c.Flags().StringVar(&model, "model", "", "the model directory") c.Flags().StringVar(&version, "version", "", "the version directory") c.Flags().StringVar(&file, "file", "", "the filename") c.Flags().StringVar(&server, "server", "127.0.0.1:9090", "admin API address to read the catalog from") c.Flags().StringVar(&base, "base", "http://localhost:8081", "the address nodes use to reach this mirror") c.Flags().BoolVar(&force, "force", false, "remove without checking the catalog") return c } func newMirrorServeCmd(dir *string) *cobra.Command { var listen, apiListen, base string var tlsc tlsconf.Config c := &cobra.Command{ Use: "serve", Short: "Serve the mirror, and an API to manage it", Long: "Two listeners, on purpose.\n\n" + "The first serves the weights themselves, read-only and unauthenticated,\n" + "because that is what eight nodes pull forty gigabytes from and putting a\n" + "client certificate in front of it would mean re-certifying every agent's\n" + "fetcher to change how models are administered. Range requests work, which\n" + "is what makes an interrupted download on a node resume rather than restart.\n\n" + "The second is the management API the control plane calls: ask the mirror to\n" + "fetch a model, watch it arrive, remove one that is retired. It can be given\n" + "the fleet's CA and then only holders of a certificate from it get in.\n\n" + "The mirror does its own downloading. A control plane that had to pull the\n" + "bytes and push them here would carry every model twice over the network,\n" + "through the one machine least able to spare the bandwidth.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { root, err := os.Stat(*dir) if err != nil || !root.IsDir() { return fmt.Errorf("%s is not a directory to serve", *dir) } if err := tlsc.Validate(); err != nil { return err } out := cmd.OutOrStdout() log := slog.Default() mgr := mirror.NewManager(*dir, log) api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/")) // http.Dir refuses paths that escape the root, which is the whole // traversal question answered by the standard library rather than // by me. files := &http.Server{ Addr: listen, Handler: readOnly(withManifest(*dir, http.FileServer(http.Dir(*dir)))), ReadHeaderTimeout: 10 * time.Second, } admin := &http.Server{ Addr: apiListen, Handler: api.Handler(), ReadHeaderTimeout: 10 * time.Second, } errs := make(chan error, 2) go func() { errs <- fmt.Errorf("serving weights: %w", files.ListenAndServe()) }() fmt.Fprintf(out, "serving %s on %s\n", *dir, listen) go func() { errs <- fmt.Errorf("serving the API: %w", serveAPI(admin, tlsc)) }() if tlsc.Enabled() { fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA) } else { // Said plainly rather than left to be discovered. An API that // can delete models and start downloads, open to anyone who can // reach the port, is a choice the operator should have made // rather than inherited from a default. fmt.Fprintf(out, "management API on %s, UNAUTHENTICATED — keep it on a trusted network\n", apiListen) } fmt.Fprintf(out, "catalog sources should point at %s\n", base) // The first failure ends it. A mirror serving bytes but refusing // management, or the reverse, is half a service pretending to be // whole, and the half that works keeps the alert quiet. return <-errs }, } c.Flags().StringVar(&listen, "listen", ":8081", "address to serve the weights on") c.Flags().StringVar(&apiListen, "api-listen", ":8082", "address to serve the management API on") c.Flags().StringVar(&base, "base", "http://localhost:8081", "the address nodes use to reach this mirror, as it should appear in catalog sources") c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from") c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate") c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key") return c } // serveAPI starts the management listener, with mutual TLS when it is // configured. // // NextProtos is cleared from what tlsconf builds. That configuration is for the // agent stream, which needs HTTP/2 for bidirectional streams and so pins "h2"; // this is an ordinary REST API, and pinning h2 would fail the handshake for // every HTTP/1.1 client — curl included — with an error that says nothing about // ALPN. Empty lets net/http offer both. func serveAPI(srv *http.Server, c tlsconf.Config) error { if !c.Enabled() { return srv.ListenAndServe() } cfg, err := c.ServerConfig() if err != nil { return err } cfg.NextProtos = nil srv.TLSConfig = cfg return srv.ListenAndServeTLS("", "") } // readOnly refuses anything but a read. // // http.FileServer already ignores other methods, but saying so here means the // property is stated rather than inherited — and a handler added later cannot // quietly make the mirror writable. func readOnly(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet && r.Method != http.MethodHead { http.Error(w, "the mirror is read-only", http.StatusMethodNotAllowed) return } h.ServeHTTP(w, r) }) } // catalogSources reads every model's source from a running control plane. func catalogSources(cmd *cobra.Command, server string) ([]string, error) { var models []adminapi.ModelView if err := newCtlClient(server).get(cmd.Context(), "/api/v1/models", &models); err != nil { return nil, err } out := make([]string, 0, len(models)) for _, m := range models { out = append(out, m.Source) } return out, nil } // withManifest answers a request for a version with the list of files in it. // // A directory otherwise gets FileServer's HTML index, and something that has to // know what a version consists of would be left parsing it. A model is // routinely several files — sharded safetensors, or a GGUF split across parts — // and a catalog entry names one source, so without a way to ask "what is in // this version" a sharded model can be mirrored and never served. // // On the byte listener rather than the management API on purpose. Fetching // weights is the one thing every node does, and making it depend on the // administrative port would undo the split that keeps that port closeable. func withManifest(root string, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { model, version, ok := versionPath(r.URL.Path) if !ok { next.ServeHTTP(w, r) return } m, err := mirror.ManifestFor(root, model, version) if err != nil { http.NotFound(w, r) return } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(m) }) } // versionPath matches "/model/version/" and nothing else. // // The trailing slash is the whole signal: "/m/v/file.gguf" is a file and // "/m/v/" is the set it belongs to. Requiring it means a source URL says which // of the two it means without a flag beside it. func versionPath(p string) (model, version string, ok bool) { if !strings.HasSuffix(p, "/") { return "", "", false } parts := strings.Split(strings.Trim(p, "/"), "/") if len(parts) != 2 || parts[0] == "" || parts[1] == "" { return "", "", false } return parts[0], parts[1], true }