From 44425193bd53469949b25f5fb07ba72dbcb6fdfb Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:12:20 +0200 Subject: [PATCH 1/5] Track the registry's new weights for served models, and move a model to them or back A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/reference.md | 93 ++++ internal/cli/model.go | 3 +- internal/cli/model_serve.go | 57 +- internal/cli/model_update.go | 119 ++++ internal/cli/server.go | 17 +- internal/cli/tui_actions.go | 56 ++ internal/cli/tui_console.go | 19 +- internal/config/config.go | 6 + internal/control/adminapi/api.go | 94 ++++ internal/control/catalog/catalog.go | 45 ++ internal/control/modelserve/modelserve.go | 34 +- internal/control/modelserve/updates.go | 569 ++++++++++++++++++++ internal/control/modelserve/updates_test.go | 264 +++++++++ internal/control/modelsvc/modelsvc.go | 32 ++ internal/docgen/reference_gen.go | 18 + 15 files changed, 1393 insertions(+), 33 deletions(-) create mode 100644 internal/cli/model_update.go create mode 100644 internal/control/modelserve/updates.go create mode 100644 internal/control/modelserve/updates_test.go diff --git a/docs/reference.md b/docs/reference.md index b8606a7..ab6c09b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -76,6 +76,7 @@ rather than repeated. | `tls` | `tlsconf.Config` | [Mirror](#mirror) | | `registry` | `string` | [Mirror](#mirror) | | `timeout` | `time.Duration` | [Mirror](#mirror) | +| `update_check` | `time.Duration` | [Mirror](#mirror) | | `name` | `string` | [Model](#model) | | `aliases` | `[]string` | [Model](#model) | | `description` | `string` | [Model](#model) | @@ -92,6 +93,7 @@ rather than repeated. | `context_length` | `int` | [Model](#model) | | `max_output` | `int` | [Model](#model) | | `capabilities` | `Capabilities` | [Model](#model) | +| `origin` | `*Origin` | [Model](#model) | | `prometheus` | `string` | [Monitoring](#monitoring) | | `window` | `time.Duration` | [Monitoring](#monitoring) | | `timeout` | `time.Duration` | [Monitoring](#monitoring) | @@ -99,6 +101,13 @@ rather than repeated. | `ib_root` | `string` | [Options](#options) | | `prefer_nvml` | `bool` | [Options](#options) | | `skip_gpus` | `bool` | [Options](#options) | +| `repo` | `string` | [Origin](#origin) | +| `revision` | `string` | [Origin](#origin) | +| `files` | `[]OriginFile` | [Origin](#origin) | +| `previous` | `*PreviousOrigin` | [Origin](#origin) | +| `path` | `string` | [OriginFile](#originfile) | +| `sha256` | `string` | [OriginFile](#originfile) | +| `bytes` | `int64` | [OriginFile](#originfile) | | `disabled` | `bool` | [Personal](#personal) | | `rpm` | `int` | [Personal](#personal) | | `tpm` | `int` | [Personal](#personal) | @@ -111,6 +120,9 @@ rather than repeated. | `match_ratio` | `float64` | [Prefix](#prefix) | | `balance_abs` | `float64` | [Prefix](#prefix) | | `balance_rel` | `float64` | [Prefix](#prefix) | +| `source` | `string` | [PreviousOrigin](#previousorigin) | +| `revision` | `string` | [PreviousOrigin](#previousorigin) | +| `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) | | `name` | `string` | [Profile](#profile) | | `engine` | `string` | [Profile](#profile) | | `gpu_class` | `string` | [Profile](#profile) | @@ -668,6 +680,15 @@ Timeout bounds a management call. It does not bound a download: the mirror answers a fetch as soon as it has started one, and the bytes keep arriving long after this deadline has passed. +### `update_check` + +`time.Duration` + +UpdateCheck is how often the registry is asked whether the files of +the models served from it have changed there: a repository can publish +new weights under the same name. Only told, never applied — moving a +model to them is `llmgw ctl model update`. 0 never asks. + ## Model ### `name` @@ -791,6 +812,19 @@ Capabilities is what this model can do beyond plain completion. Takes a [Capabilities](#capabilities) block. +### `origin` + +`*Origin` + +Origin is the registry repository the weights came from, at which +commit, and each file's checksum there. Written when a model is served +from the registry; for one that was not, worked out from the mirror's +record of where it fetched the file. It is what lets the control plane +say that the repository has published different weights since, and +move the model to them. + +Takes a [Origin](#origin) block. + ## Monitoring ### `prometheus` @@ -845,6 +879,51 @@ is used, is the more exact source of the peers. SkipGPUs omits GPU discovery. A node without nvidia-smi is already found to have none, so this is for one whose driver is broken. +## Origin + +### `repo` + +`string` + +Repo is the repository, owner/name. + +### `revision` + +`string` + +Revision is the commit the files were fetched at. + +### `files` + +`[]OriginFile` + +Files are the weights fetched from it, with what the registry +published as each one's sha256. A new commit that leaves them alone — +a README edited — is not an update. + +### `previous` + +`*PreviousOrigin` + +Previous is what the last update replaced, kept so it can be gone back +to while its files are still on the mirror. + +Takes a [PreviousOrigin](#previousorigin) block. + +## OriginFile + +### `path` + +`string` + +### `sha256` + +`string` + +### `bytes` + +`int64` + ## Personal ### `disabled` @@ -910,6 +989,20 @@ before affinity is abandoned. `float64` +## PreviousOrigin + +### `source` + +`string` + +### `revision` + +`string` + +### `files` + +`[]OriginFile` + ## Profile ### `name` diff --git a/internal/cli/model.go b/internal/cli/model.go index f60b854..94e4f48 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -35,7 +35,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), - newModelServeCmd(client)) + newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), + newModelRollbackCmd(client)) return c } diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 174913b..8c79501 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -38,32 +38,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { return err } - out := cmd.OutOrStdout() - fmt.Fprintf(out, "job %s\n", job.ID) - shown := map[string]string{} - for { - for _, s := range job.Steps { - line := s.State + " " + s.Detail - if shown[s.Name] == line { - continue - } - shown[s.Name] = line - mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok", - modelserve.Failed: "!!"}[s.State] - fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail) - } - switch job.State { - case modelserve.Done: - fmt.Fprintf(out, "\n%s is served.\n", job.Model) - return nil - case modelserve.Failed: - return errors.New(job.Error) - } - time.Sleep(2 * time.Second) - if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil { - return err - } - } + return followJob(cmd, client, job, "is served") }, } f := c.Flags() @@ -79,3 +54,33 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use") return c } + +// followJob shows a job's steps as they happen, until it ends. +func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, doneSays string) error { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "job %s\n", job.ID) + shown := map[string]string{} + for { + for _, s := range job.Steps { + line := s.State + " " + s.Detail + if shown[s.Name] == line { + continue + } + shown[s.Name] = line + mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok", + modelserve.Failed: "!!", modelserve.Skipped: "--"}[s.State] + fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail) + } + switch job.State { + case modelserve.Done: + fmt.Fprintf(out, "\n%s %s.\n", job.Model, doneSays) + return nil + case modelserve.Failed: + return errors.New(job.Error) + } + time.Sleep(2 * time.Second) + if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil { + return err + } + } +} diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go new file mode 100644 index 0000000..4111663 --- /dev/null +++ b/internal/cli/model_update.go @@ -0,0 +1,119 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" +) + +func newModelUpdatesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + var check bool + c := &cobra.Command{ + Use: "updates [MODEL]", + Short: "Show which models' repositories have published new weights", + Long: "A model served from the registry remembers the repository and commit its\n" + + "files came from, and the control plane asks the registry every\n" + + "mirror.update_check (six hours by default) whether those files have\n" + + "changed there. A new commit that leaves them alone is not an update.\n\n" + + "--check asks now rather than showing the last answer.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var list []modelserve.UpdateInfo + if check || len(args) == 1 { + req := map[string]string{} + if len(args) == 1 { + req["model"] = args[0] + } + body, _ := json.Marshal(req) + if err := client().post(cmd.Context(), "/api/v1/updates/check", body, &list); err != nil { + return err + } + } else if err := client().get(cmd.Context(), "/api/v1/updates", &list); err != nil { + return err + } + if *asJSON { + return emitJSON(list) + } + if len(list) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no model has an origin to compare yet; one is checked a few minutes after the control plane starts") + return nil + } + w := tabTo(cmd.OutOrStdout()) + fmt.Fprintln(w, "MODEL\tREPOSITORY\tRUNS\tLATEST\tUPDATE\tCHECKED") + for _, u := range list { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", u.Model, u.Repo, catalog.ShortRevision(u.Current), + catalog.ShortRevision(u.Latest), updateCell(u), ago(u.CheckedAt, time.Now())) + } + if err := w.Flush(); err != nil { + return err + } + for _, u := range list { + if u.Available { + fmt.Fprintf(cmd.OutOrStdout(), "\nTo move one to its new weights: llmgw ctl model update MODEL\n") + break + } + } + return nil + }, + } + c.Flags().BoolVar(&check, "check", false, "ask the registry now") + return c +} + +// updateCell says what an update would be, or why there is none. +func updateCell(u modelserve.UpdateInfo) string { + switch { + case u.Err != "": + return "could not ask: " + truncate(u.Err, 60) + case u.Available: + return fmt.Sprintf("yes: %d file(s), %s", len(u.Changed), bytesShort(u.Bytes)) + case len(u.Missing) > 0: + return "the repository no longer has " + strings.Join(u.Missing, ", ") + case u.Latest != "" && u.Latest != u.Current: + return "no: new commit, same weights" + } + return "-" +} + +func newModelUpdateCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "update MODEL", + Short: "Move a model to the weights its repository publishes now", + Long: "The new files are fetched into the mirror beside the ones in use and\n" + + "checked against the registry's checksums and the nodes' llama.cpp; then\n" + + "the catalog names them and the replicas are replaced one at a time, each\n" + + "new one up before an old one stops, and one is asked to say hello.\n\n" + + "The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" + + "back to them. A model from server.yaml is changed there instead.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var job modelserve.Job + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil { + return err + } + return followJob(cmd, client, job, "runs the new weights") + }, + } +} + +func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "rollback MODEL", + Short: "Move a model back to the weights its last update replaced", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var out map[string]string + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/rollback", []byte("{}"), &out); err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), out["result"]) + return nil + }, + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 4688a26..d820e0b 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -128,6 +128,7 @@ func newServerCmd() *cobra.Command { var agentHandler http.Handler var agentPath string var admin *control.Controller + updateMetrics := &modelserve.UpdateCollector{} // The store's cluster, when it is one; read by the metrics and // by `ctl status`. var storeHA *pgha.Collector @@ -183,6 +184,11 @@ func newServerCmd() *cobra.Command { if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil { return err } + // Whether models' repositories have new weights; filled in + // once the mirror is set up, below. + if err := rt.RegisterMetrics(updateMetrics); err != nil { + return err + } // A store of several hosts is a replicated cluster: what state // it is in is published beside everything else, read from its // Patroni. One host is a single PostgreSQL, which has none. @@ -321,10 +327,17 @@ func newServerCmd() *cobra.Command { // One command from a registry repository to a replica that // answers: `llmgw ctl model serve`. if adminAPI != nil && catalogSvc != nil { - adminAPI.SetServe(modelserve.New(modelserve.Deps{ + serve := modelserve.New(modelserve.Deps{ Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin, LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log, - })) + // Updates: the catalog in force, the service that + // changes it, and what replaces replicas. + Current: func() []catalog.Model { return admin.Catalog().Models() }, + Models: catalogSvc, Restart: admin.Reconciler().Restart, + }) + adminAPI.SetServe(serve) + updateMetrics.Set(serve) + go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck) } log.Info("model mirror configured", "api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled()) diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 47cd17f..05a0784 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -418,6 +419,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog { }) }}, } + items = append(items, m.updateItems(md)...) if md.Parked { items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd { return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil) @@ -674,3 +676,57 @@ func (m *consoleModel) doEntry() tea.Cmd { } return nil } + +// updateItems are what can be done about a model's repository publishing new +// weights: move to them, ask now, or go back. +func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { + if md.Origin == nil { + return nil + } + server := "--server=" + m.client.base + var items []menuItem + if u := md.Update; u != nil && u.Available { + items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)), + func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Update " + md.Name, + text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+ + "They are fetched into the mirror beside the ones in use and checked, then the\n"+ + "replicas are replaced one at a time. The old files stay, to go back to.", + u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")), + yes: func(m *consoleModel) tea.Cmd { return m.runAfter("ctl", server, "model", "update", md.Name) }} + return nil + }}) + } + items = append(items, menuItem{"Ask the registry for new weights now", func(m *consoleModel) tea.Cmd { + return m.call("New weights for "+md.Name, http.MethodPost, "/api/v1/updates/check", map[string]string{"model": md.Name}, + func(raw json.RawMessage) string { + var us []modelserve.UpdateInfo + if json.Unmarshal(raw, &us) != nil || len(us) == 0 { + return string(raw) + } + u := us[0] + if u.Available { + return fmt.Sprintf("%s has new weights at %s: %s, %s.", u.Repo, catalog.ShortRevision(u.Latest), + strings.Join(u.Changed, ", "), bytesShort(u.Bytes)) + } + return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u)) + }) + }}) + if prev := md.Origin.Previous; prev != "" { + items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Go back on " + md.Name, + text: "It runs the weights its last update replaced, " + catalog.ShortRevision(prev) + + ", if they are still on the mirror;\nits replicas are replaced one at a time.", + yes: func(m *consoleModel) tea.Cmd { + return m.call("Go back on "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/rollback", + map[string]string{}, func(raw json.RawMessage) string { + var out map[string]string + _ = json.Unmarshal(raw, &out) + return out["result"] + "." + }) + }} + return nil + }}) + } + return items +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index c48022a..ce7e888 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -19,6 +19,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -959,13 +960,17 @@ func (m consoleModel) panel() (title, body, keys string) { if len(md.Warnings) > 0 { warn = sWarn.Render("!") } + upd := "" + if md.Update != nil && md.Update.Available { + upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest)) + } rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class, - strings.Join(md.Profiles, ","), warn}) + strings.Join(md.Profiles, ","), upd, warn}) } if m.inPanel { keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave" } - return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", ""}, rows, m.row, pw, m.inPanel), keys + return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys case cReplicas: var rows [][]string for _, r := range m.data.replicas { @@ -1174,6 +1179,16 @@ func (m consoleModel) modelDetail(md adminapi.ModelView) string { kv("profiles", strings.Join(md.Profiles, ", ")) kv("aliases", strings.Join(md.Aliases, ", ")) kv("source", md.Source) + if o := md.Origin; o != nil { + from := o.Repo + " at " + catalog.ShortRevision(o.Revision) + if o.Previous != "" { + from += ", updated from " + catalog.ShortRevision(o.Previous) + } + kv("from", from) + } + if u := md.Update; u != nil { + kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now()))) + } if md.MemoryGB > 0 { kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB)) } diff --git a/internal/config/config.go b/internal/config/config.go index a01980b..bf09a06 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -177,6 +177,11 @@ type Mirror struct { // mirror answers a fetch as soon as it has started one, and the bytes keep // arriving long after this deadline has passed. Timeout time.Duration `koanf:"timeout"` + // UpdateCheck is how often the registry is asked whether the files of + // the models served from it have changed there: a repository can publish + // new weights under the same name. Only told, never applied — moving a + // model to them is `llmgw ctl model update`. 0 never asks. + UpdateCheck time.Duration `koanf:"update_check"` } // Landing configures the public page at "/". @@ -481,6 +486,7 @@ func Default() Server { Listen: ":8080", AdminListen: "127.0.0.1:9090", AgentListen: ":7443", + Mirror: Mirror{UpdateCheck: 6 * time.Hour}, Control: Control{ Interval: 5 * time.Second, BaseBackoff: 10 * time.Second, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3a18611..a32d851 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -121,6 +121,10 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/restart", a.restart) mux.HandleFunc("POST /api/v1/models/serve", a.serveModel) mux.HandleFunc("GET /api/v1/jobs/{id}", a.job) + mux.HandleFunc("GET /api/v1/updates", a.listUpdates) + mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) + mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) + mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -336,6 +340,19 @@ type ModelView struct { Adapters []catalog.Adapter `json:"adapters,omitempty"` Preempted int `json:"preempted,omitempty"` HeldFor string `json:"held_for,omitempty"` + // Origin is the registry repository its weights came from, and at which + // commit; Update what that repository had when it was last asked. + Origin *OriginView `json:"origin,omitempty"` + Update *modelserve.UpdateInfo `json:"update,omitempty"` +} + +// OriginView is where a model's weights were published. +type OriginView struct { + Repo string `json:"repo"` + Revision string `json:"revision"` + // Previous is the commit the last update replaced, which it can go back + // to. + Previous string `json:"previous,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -586,6 +603,10 @@ func (a *API) Models() []ModelView { } measured[rep.Model] = max(measured[rep.Model], b) } + var updates map[string]modelserve.UpdateInfo + if a.serve != nil { + updates = a.serve.Updates() + } out := make([]ModelView, 0) for _, m := range a.ctl.Catalog().Models() { h := health[m.Name] @@ -605,6 +626,15 @@ func (a *API) Models() []ModelView { if len(m.Profiles) > 0 { v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores } + if o := m.Origin; o != nil { + v.Origin = &OriginView{Repo: o.Repo, Revision: o.Revision} + if o.Previous != nil { + v.Origin.Previous = o.Previous.Revision + } + } + if u, ok := updates[m.Name]; ok { + v.Update = &u + } v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30) v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB) out = append(out, v) @@ -1426,6 +1456,70 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, job) } +func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) { + out := make([]modelserve.UpdateInfo, 0) + if a.serve != nil { + for _, u := range a.serve.Updates() { + out = append(out, u) + } + } + slices.SortFunc(out, func(x, y modelserve.UpdateInfo) int { return strings.Compare(x.Model, y.Model) }) + writeJSON(w, http.StatusOK, out) +} + +// CheckRequest asks the registry about one model now, or every one. +type CheckRequest struct { + Model string `json:"model,omitempty"` +} + +func (a *API) checkUpdates(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + var req CheckRequest + _ = json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req) + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute) + defer cancel() + if req.Model != "" { + u, err := a.serve.CheckModel(ctx, req.Model) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, []modelserve.UpdateInfo{u}) + return + } + a.serve.CheckUpdates(ctx) + a.listUpdates(w, r) +} + +func (a *API) updateModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusAccepted, job) +} + +func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + msg, err := a.serve.Rollback(r.Context(), r.PathValue("name"), actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"result": msg}) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 70c599e..710e5c5 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -194,6 +194,51 @@ type Model struct { // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` + + // Origin is the registry repository the weights came from, at which + // commit, and each file's checksum there. Written when a model is served + // from the registry; for one that was not, worked out from the mirror's + // record of where it fetched the file. It is what lets the control plane + // say that the repository has published different weights since, and + // move the model to them. + Origin *Origin `json:"origin,omitempty" koanf:"origin"` +} + +// Origin is where a model's weights were published. +type Origin struct { + // Repo is the repository, owner/name. + Repo string `json:"repo" koanf:"repo"` + // Revision is the commit the files were fetched at. + Revision string `json:"revision" koanf:"revision"` + // Files are the weights fetched from it, with what the registry + // published as each one's sha256. A new commit that leaves them alone — + // a README edited — is not an update. + Files []OriginFile `json:"files,omitempty" koanf:"files"` + // Previous is what the last update replaced, kept so it can be gone back + // to while its files are still on the mirror. + Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"` +} + +// OriginFile is one weights file of an origin. +type OriginFile struct { + Path string `json:"path" koanf:"path"` + SHA256 string `json:"sha256,omitempty" koanf:"sha256"` + Bytes int64 `json:"bytes,omitempty" koanf:"bytes"` +} + +// PreviousOrigin is a model's source before an update, and where it came from. +type PreviousOrigin struct { + Source string `json:"source" koanf:"source"` + Revision string `json:"revision" koanf:"revision"` + Files []OriginFile `json:"files,omitempty" koanf:"files"` +} + +// ShortRevision is a commit as a person writes it. +func ShortRevision(rev string) string { + if len(rev) > 8 { + return rev[:8] + } + return rev } // Capabilities describes what a client may ask a model for. diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 6d22a2d..3ef4807 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -71,6 +71,12 @@ type Deps struct { // ReadyTimeout bounds the wait for a replica: the node fetching the // weights from the mirror and the engine loading them. ReadyTimeout time.Duration + + // For updates: the catalog in force, the service that changes it, and + // what replaces replicas. Without them updates are not offered. + Current func() []catalog.Model + Models Models + Restart func(ids ...string) error } // Request is what to serve. @@ -116,7 +122,10 @@ type Step struct { // Job is one model being served, and where it stands. type Job struct { - ID string `json:"id"` + ID string `json:"id"` + // Kind is "update" for a model moved to new weights, empty for one + // served. + Kind string `json:"kind,omitempty"` Request Request `json:"request"` State string `json:"state"` Error string `json:"error,omitempty"` @@ -135,6 +144,8 @@ type Server struct { mu sync.Mutex jobs map[string]*Job seq int + // updates is what the last checks of the registry found, by model. + updates map[string]UpdateInfo } // New builds a server. @@ -151,7 +162,7 @@ func New(d Deps) *Server { if d.Log == nil { d.Log = slog.Default() } - return &Server{deps: d, jobs: map[string]*Job{}} + return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}} } var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) @@ -297,6 +308,7 @@ func (s *Server) run(j *Job) { Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, }}, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, + Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { @@ -388,8 +400,26 @@ func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSe version := repo.ShortRevision() s.step(j, "mirror", Running, "asking the mirror for "+set.Name) var base string + // Files already there from this commit, checksums matching, are not + // fetched again: a model moved back and then forward again. + have := map[string]bool{} + if got, err := s.deps.Mirror.List(ctx); err == nil { + for _, st := range got.Models { + if st.Model == name && st.Version == version && st.State == mirror.Present { + for _, f := range set.Files { + if mirror.LastPathElement(f.Path) == st.File && (f.SHA256 == "" || strings.EqualFold(f.SHA256, st.SHA256)) { + have[st.File] = true + base = mirror.Entry{Model: name, Version: version, File: st.File}.URL(got.Base) + } + } + } + } + } for _, f := range set.Files { file := mirror.LastPathElement(f.Path) + if have[file] { + continue + } resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{ Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file, }) diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go new file mode 100644 index 0000000..cc446e8 --- /dev/null +++ b/internal/control/modelserve/updates.go @@ -0,0 +1,569 @@ +package modelserve + +import ( + "context" + "errors" + "fmt" + "net/url" + "slices" + "strings" + "sync/atomic" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "github.com/prometheus/client_golang/prometheus" +) + +// Updates: a repository publishes new weights under the same name — a fixed +// chat template, a better quantisation — and nothing told anyone. A model +// served from the registry now remembers where it came from; this asks the +// registry, every so often, whether the files it runs have changed there, +// and moves a model to them on request: fetched into the mirror beside the +// old ones, checked, swapped in, its replicas replaced one at a time, and +// the old files kept so it can go back. + +// Models is the catalog as an update changes it. +type Models interface { + Declared(name string) bool + SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) + SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error +} + +// UpdateInfo is what the registry has for a model against what it runs. +type UpdateInfo struct { + Model string `json:"model"` + Repo string `json:"repo,omitempty"` + // Current is the commit the model runs, Latest the repository's now. + Current string `json:"current,omitempty"` + Latest string `json:"latest,omitempty"` + // Changed are the model's files whose weights differ at Latest, and + // Missing those Latest no longer has at all. + Changed []string `json:"changed,omitempty"` + Missing []string `json:"missing,omitempty"` + // Bytes is what an update downloads. + Bytes int64 `json:"bytes,omitempty"` + // Available is true when the model can be moved to Latest. + Available bool `json:"available"` + // Previous is the commit an update replaced, which it can go back to. + Previous string `json:"previous,omitempty"` + CheckedAt time.Time `json:"checked_at"` + Err string `json:"error,omitempty"` +} + +// place is where a model's weights sit on the mirror. +type place struct { + model, version string + files []mirror.Status +} + +// placeOf finds a source on the mirror: a file, or a directory of shards. +func placeOf(source string, list mirror.ModelsResponse) (place, bool) { + base := strings.TrimSuffix(list.Base, "/") + "/" + if list.Base == "" || !strings.HasPrefix(source, base) { + return place{}, false + } + parts := strings.SplitN(strings.TrimPrefix(source, base), "/", 3) + if len(parts) < 2 { + return place{}, false + } + p := place{model: parts[0], version: parts[1]} + file := "" + if len(parts) == 3 { + file = parts[2] + } + for _, st := range list.Models { + if st.Model == p.model && st.Version == p.version && (file == "" || st.File == file) { + p.files = append(p.files, st) + } + } + return p, len(p.files) > 0 +} + +// parseResolve reads a registry download URL: base/owner/name/resolve/rev/path. +func parseResolve(u string) (repo, rev, path string, ok bool) { + pu, err := url.Parse(u) + if err != nil { + return "", "", "", false + } + i := strings.Index(pu.Path, "/resolve/") + if i < 0 { + return "", "", "", false + } + repo = strings.Trim(pu.Path[:i], "/") + if strings.Count(repo, "/") != 1 { + return "", "", "", false + } + rev, path, ok = strings.Cut(pu.Path[i+len("/resolve/"):], "/") + return repo, rev, path, ok && rev != "" && path != "" +} + +// inferOrigin works a model's origin out from what the mirror recorded of +// where it fetched each file, for a model served before origins were kept. +func inferOrigin(p place) (*catalog.Origin, bool) { + var o catalog.Origin + for _, st := range p.files { + repo, rev, path, ok := parseResolve(st.Source) + if !ok || (o.Repo != "" && (repo != o.Repo || rev != o.Revision)) { + return nil, false + } + o.Repo, o.Revision = repo, rev + o.Files = append(o.Files, catalog.OriginFile{Path: path, SHA256: st.SHA256, Bytes: st.BytesDone}) + } + return &o, o.Repo != "" +} + +func originFiles(set hf.FileSet) []catalog.OriginFile { + out := make([]catalog.OriginFile, 0, len(set.Files)) + for _, f := range set.Files { + out = append(out, catalog.OriginFile{Path: f.Path, SHA256: f.SHA256, Bytes: f.Bytes}) + } + return out +} + +// originOf is a model's origin: recorded, or worked out from the mirror and +// then recorded. +func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin { + if m.Origin != nil && m.Origin.Repo != "" { + return m.Origin + } + p, ok := placeOf(m.Source, list) + if !ok { + return nil + } + o, ok := inferOrigin(p) + if !ok { + return nil + } + if s.deps.Models != nil { + if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil { + s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err) + } + } + return o +} + +// compare says what the registry's repository has against an origin. +func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { + u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()} + if o.Previous != nil { + u.Previous = o.Previous.Revision + } + if repo.Revision == o.Revision { + return u + } + for _, f := range o.Files { + i := slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path }) + if i < 0 { + u.Missing = append(u.Missing, f.Path) + continue + } + g := repo.Files[i] + if (f.SHA256 != "" && g.SHA256 != "" && !strings.EqualFold(f.SHA256, g.SHA256)) || + (f.SHA256 == "" && f.Bytes != 0 && f.Bytes != g.Bytes) { + u.Changed = append(u.Changed, f.Path) + u.Bytes += g.Bytes + } + } + u.Available = len(u.Changed) > 0 && len(u.Missing) == 0 + return u +} + +// CheckUpdates asks the registry about every model with an origin. +func (s *Server) CheckUpdates(ctx context.Context) { + if s.deps.Current == nil { + return + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + s.deps.Log.Warn("checking for model updates: the mirror", "err", err) + return + } + seen := map[string]bool{} + for _, m := range s.deps.Current() { + o := s.originOf(ctx, m, list) + if o == nil { + continue + } + seen[m.Name] = true + s.checkOne(ctx, m, o) + } + s.mu.Lock() + for name := range s.updates { + if !seen[name] { + delete(s.updates, name) + } + } + s.mu.Unlock() +} + +func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origin) UpdateInfo { + repo, err := s.deps.Registry.Repo(ctx, o.Repo) + u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, CheckedAt: time.Now()} + if err != nil { + u.Err = err.Error() + } else { + u = compare(m, o, repo) + if u.Available { + s.deps.Log.Info("a model's repository has new weights", "model", m.Name, "repo", o.Repo, + "current", catalog.ShortRevision(o.Revision), "latest", catalog.ShortRevision(repo.Revision)) + } + } + s.mu.Lock() + s.updates[m.Name] = u + s.mu.Unlock() + return u +} + +// CheckModel asks the registry about one model now. +func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error) { + m, err := s.current(name) + if err != nil { + return UpdateInfo{}, err + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return UpdateInfo{}, fmt.Errorf("the mirror: %w", err) + } + o := s.originOf(ctx, m, list) + if o == nil { + return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name) + } + return s.checkOne(ctx, m, o), nil +} + +// Updates is what the last checks found, by model. +func (s *Server) Updates() map[string]UpdateInfo { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]UpdateInfo, len(s.updates)) + for k, v := range s.updates { + out[k] = v + } + return out +} + +// WatchUpdates checks every so often until ctx ends; zero never checks. +func (s *Server) WatchUpdates(ctx context.Context, every time.Duration) { + if every <= 0 { + return + } + // A little after starting, when the catalog is in force and the mirror + // answering, rather than in the middle of starting. + wait := 2 * time.Minute + for { + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + s.CheckUpdates(ctx) + wait = every + } +} + +func (s *Server) current(name string) (catalog.Model, error) { + if s.deps.Current == nil { + return catalog.Model{}, errors.New("this control plane cannot read its catalog here") + } + for _, m := range s.deps.Current() { + if m.Name == name { + return m, nil + } + } + return catalog.Model{}, fmt.Errorf("no model called %s", name) +} + +// StartUpdate moves a model to its repository's latest weights, as a job. +func (s *Server) StartUpdate(name, by string) (Job, error) { + if s.deps.Models == nil || s.deps.Restart == nil { + return Job{}, errors.New("updating needs the store, which this control plane lacks") + } + if _, err := s.current(name); err != nil { + return Job{}, err + } + if s.deps.Models.Declared(name) { + return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+ + "or they would revert on the next restart", name) + } + s.mu.Lock() + for _, j := range s.jobs { + if j.Kind == "update" && j.Model == name && j.State == Running { + s.mu.Unlock() + return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID) + } + } + s.seq++ + j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name, + Request: Request{Name: name, By: by}, State: Running, Started: time.Now()} + s.jobs[j.ID] = j + s.mu.Unlock() + go s.runUpdate(j) + return s.snapshot(j), nil +} + +func (s *Server) runUpdate(j *Job) { + ctx := context.Background() + name := j.Model + + // 1. The registry: which files, at which commit. + s.step(j, "registry", Running, "reading where "+name+" came from") + m, err := s.current(name) + if err != nil { + s.fail(j, "registry", err) + return + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + s.fail(j, "registry", fmt.Errorf("the mirror: %w", err)) + return + } + o := s.originOf(ctx, m, list) + p, onMirror := placeOf(m.Source, list) + if o == nil || !onMirror { + s.fail(j, "registry", fmt.Errorf("%s did not come from the registry through the mirror", name)) + return + } + repo, err := s.deps.Registry.Repo(ctx, o.Repo) + if err != nil { + s.fail(j, "registry", err) + return + } + s.mu.Lock() + j.Request.Repo = repo.ID + s.mu.Unlock() + u := compare(m, o, repo) + switch { + case repo.Revision == o.Revision: + s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision))) + return + case len(u.Missing) > 0: + s.fail(j, "registry", fmt.Errorf("%s at %s no longer has %s; choose one of its files and serve it under a new name", + repo.ID, repo.ShortRevision(), strings.Join(u.Missing, ", "))) + return + case len(u.Changed) == 0: + s.fail(j, "registry", fmt.Errorf("%s has a newer commit, %s, but the files %s runs are the same in it", + repo.ID, repo.ShortRevision(), name)) + return + case repo.Gated.Is(): + s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", repo.ID)) + return + } + set := hf.FileSet{} + for _, f := range o.Files { + g := repo.Files[slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })] + set.Files = append(set.Files, g) + set.Bytes += g.Bytes + } + set.Name, set.Shards = set.Files[0].Path, len(set.Files) + s.step(j, "registry", Done, fmt.Sprintf("%s: %s → %s, %d of %d file(s) changed, %.1f GB", + repo.ID, catalog.ShortRevision(o.Revision), repo.ShortRevision(), len(u.Changed), len(o.Files), gb(set.Bytes))) + + // 2. The mirror, beside the files in use. + source, err := s.mirror(ctx, j, repo, set, p.model) + if err != nil { + s.fail(j, "mirror", err) + return + } + + // 3. Whether the nodes load it, and whether it still fits what is + // reserved for it. + s.step(j, "check", Running, "reading its header from the mirror") + detail := "a set of shards; not read" + if !strings.HasSuffix(source, "/") { + v := gguf.Check(ctx, source, s.deps.LlamaVersion) + if !v.Servable { + s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) + return + } + detail = orUnknown(v.Header.Architecture) + if len(m.Profiles) > 0 { + ctxPool := 16384 + for i, a := range m.Profiles[0].Args { + if a == "--ctx-size" && i+1 < len(m.Profiles[0].Args) { + fmt.Sscan(m.Profiles[0].Args[i+1], &ctxPool) + } + } + need := int((v.Header.MemoryBytes(set.Bytes, ctxPool) + 1<<30 - 1) >> 30) + if have := m.Profiles[0].MinMemoryGB; need > have && have > 0 { + detail += fmt.Sprintf("; needs about %d GB and %d is reserved: raise it if replicas fail", need, have) + } + } + } + s.step(j, "check", Done, detail) + + // 4. The catalog: the new source, and the old one kept to go back to. + s.step(j, "catalog", Running, "moving "+name+" to "+repo.ShortRevision()) + next := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set), + Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: o.Revision, Files: o.Files}} + if _, err := s.deps.Models.SetSource(ctx, name, source, next, j.Request.By); err != nil { + s.fail(j, "catalog", err) + return + } + s.step(j, "catalog", Done, "the catalog names "+source) + + // 5. The replicas, replaced one at a time. + if err := s.replace(ctx, j, name); err != nil { + s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back to %s", err, name, + catalog.ShortRevision(o.Revision))) + return + } + s.mu.Lock() + s.updates[name] = UpdateInfo{Model: name, Repo: repo.ID, Current: repo.Revision, Latest: repo.Revision, + Previous: o.Revision, CheckedAt: time.Now()} + s.mu.Unlock() + s.finish(j) +} + +// replace restarts a model's replicas onto the catalog in force and waits +// for the new ones, then asks one to answer. +func (s *Server) replace(ctx context.Context, j *Job, name string) error { + var old []string + running := 0 + for _, r := range s.deps.Fleet.Replicas() { + if r.Model != name { + continue + } + running++ + if r.State == reconciler.Ready { + old = append(old, r.ID) + } + } + if running == 0 { + s.step(j, "replicas", Skipped, "none running; the next to start loads the new weights") + return nil + } + if len(old) == 0 { + s.step(j, "replicas", Skipped, "none ready; those starting load the new weights when they next start") + return nil + } + if err := s.deps.Restart(old...); err != nil { + return err + } + s.step(j, "replicas", Running, fmt.Sprintf("replacing %d, one at a time", len(old))) + deadline := time.Now().Add(s.deps.ReadyTimeout) + for { + left, ready := 0, []reconciler.Replica{} + for _, r := range s.deps.Fleet.Replicas() { + if r.Model != name { + continue + } + if slices.Contains(old, r.ID) { + left++ + } else if r.State == reconciler.Ready { + ready = append(ready, r) + } + } + for _, h := range s.deps.Fleet.ModelHealth() { + if h.Model == name && h.Parked { + return fmt.Errorf("the new weights failed to start %d times: %s", h.Failures, h.Reason) + } + } + if left == 0 && len(ready) > 0 { + s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old))) + s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello") + reply, tps, err := s.smoke(ctx, ready[0], name) + if err != nil { + s.step(j, "smoke test", Failed, err.Error()) + return err + } + s.mu.Lock() + j.Reply, j.TokensPerSecond = reply, tps + s.mu.Unlock() + s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply)) + return nil + } + s.step(j, "replicas", Running, fmt.Sprintf("%d of %d replaced, %d new ready", len(old)-left, len(old), len(ready))) + if time.Now().After(deadline) { + return fmt.Errorf("the replicas were not replaced after %s", s.deps.ReadyTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.deps.Poll): + } + } +} + +// Rollback moves a model back to what its last update replaced, if those +// files are still on the mirror, and restarts its replicas onto them. The +// update is kept as the one to go forward to again. +func (s *Server) Rollback(ctx context.Context, name, by string) (string, error) { + if s.deps.Models == nil || s.deps.Restart == nil { + return "", errors.New("going back needs the store, which this control plane lacks") + } + m, err := s.current(name) + if err != nil { + return "", err + } + if m.Origin == nil || m.Origin.Previous == nil { + return "", fmt.Errorf("%s has not been updated, so there is nothing to go back to", name) + } + prev := m.Origin.Previous + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return "", fmt.Errorf("the mirror: %w", err) + } + if _, ok := placeOf(prev.Source, list); !ok { + return "", fmt.Errorf("the files of %s are no longer on the mirror", catalog.ShortRevision(prev.Revision)) + } + back := &catalog.Origin{Repo: m.Origin.Repo, Revision: prev.Revision, Files: prev.Files, + Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: m.Origin.Revision, Files: m.Origin.Files}} + if _, err := s.deps.Models.SetSource(ctx, name, prev.Source, back, by); err != nil { + return "", err + } + var ids []string + for _, r := range s.deps.Fleet.Replicas() { + if r.Model == name && r.State == reconciler.Ready { + ids = append(ids, r.ID) + } + } + msg := fmt.Sprintf("%s runs %s again", name, catalog.ShortRevision(prev.Revision)) + if len(ids) > 0 { + if err := s.deps.Restart(ids...); err != nil { + return "", err + } + msg += fmt.Sprintf("; %d replica(s) are being replaced", len(ids)) + } + s.mu.Lock() + if u, ok := s.updates[name]; ok { + u.Current, u.Previous = prev.Revision, m.Origin.Revision + u.Available = u.Latest != "" && u.Latest != prev.Revision + s.updates[name] = u + } + s.mu.Unlock() + return msg, nil +} + +// UpdateCollector publishes llmgw_model_update_available, 1 for each model +// whose repository has new weights. Registered when the metrics are, before +// there is a server to read: Set gives it one. +type UpdateCollector struct{ s atomic.Pointer[Server] } + +var updateDesc = prometheus.NewDesc("llmgw_model_update_available", + "1 when the registry repository a model was served from has published different weights for its files.", + []string{"model", "repo"}, nil) + +// Set gives the collector the server whose checks it reports. +func (c *UpdateCollector) Set(s *Server) { c.s.Store(s) } + +// Describe implements prometheus.Collector. +func (c *UpdateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- updateDesc } + +// Collect implements prometheus.Collector. +func (c *UpdateCollector) Collect(ch chan<- prometheus.Metric) { + s := c.s.Load() + if s == nil { + return + } + for _, u := range s.Updates() { + v := 0.0 + if u.Available { + v = 1 + } + ch <- prometheus.MustNewConstMetric(updateDesc, prometheus.GaugeValue, v, u.Model, u.Repo) + } +} diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go new file mode 100644 index 0000000..104207f --- /dev/null +++ b/internal/control/modelserve/updates_test.go @@ -0,0 +1,264 @@ +package modelserve + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +const ( + oldRev = "abc123456789aaaa" + newRev = "def987654321bbbb" +) + +type movingRegistry struct { + mu sync.Mutex + repo hf.Repo +} + +func (r *movingRegistry) Repo(context.Context, string) (hf.Repo, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.repo, nil +} + +// updMirror is a mirror that records where it fetched each file from. +type updMirror struct { + mu sync.Mutex + base string + held []mirror.Status + sums map[string]string +} + +func (m *updMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.held = append(m.held, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, Source: r.Source, + State: mirror.Present, BytesDone: 1 << 30, SHA256: m.sums[r.Version+"/"+r.File]}) + return mirror.FetchResponse{URL: mirror.Entry{Model: r.Model, Version: r.Version, File: r.File}.URL(m.base)}, nil +} + +func (m *updMirror) List(context.Context) (mirror.ModelsResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + return mirror.ModelsResponse{Base: m.base, Models: append([]mirror.Status{}, m.held...)}, nil +} + +type updModels struct { + mu sync.Mutex + models map[string]catalog.Model + declared map[string]bool +} + +func (c *updModels) Declared(n string) bool { return c.declared[n] } +func (c *updModels) SetSource(_ context.Context, n, src string, o *catalog.Origin, _ string) (catalog.Model, error) { + c.mu.Lock() + defer c.mu.Unlock() + m := c.models[n] + m.Source, m.Origin = src, o + c.models[n] = m + return m, nil +} +func (c *updModels) SetOrigin(_ context.Context, n string, o *catalog.Origin) error { + c.mu.Lock() + defer c.mu.Unlock() + m := c.models[n] + m.Origin = o + c.models[n] = m + return nil +} +func (c *updModels) all() []catalog.Model { + c.mu.Lock() + defer c.mu.Unlock() + var out []catalog.Model + for _, m := range c.models { + out = append(out, m) + } + return out +} + +// restartFleet replaces a restarted replica with a new one, ready. +type restartFleet struct { + mu sync.Mutex + reps []reconciler.Replica + engine string + seq int +} + +func (f *restartFleet) Replicas() []reconciler.Replica { + f.mu.Lock() + defer f.mu.Unlock() + return append([]reconciler.Replica{}, f.reps...) +} +func (f *restartFleet) ModelHealth() []reconciler.ModelHealth { return nil } +func (f *restartFleet) EngineKey(string) string { return "" } +func (f *restartFleet) Restart(ids ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + for _, id := range ids { + for i, r := range f.reps { + if r.ID == id { + f.seq++ + f.reps[i] = reconciler.Replica{ID: r.Model + "-new" + string(rune('0'+f.seq)), Model: r.Model, + State: reconciler.Ready, Endpoint: f.engine} + } + } + } + return nil +} + +func updSetup(t *testing.T) (*Server, *updMirror, *updModels, *movingRegistry, *restartFleet) { + t.Helper() + header := ggufHeader() + files := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(header) })) + t.Cleanup(files.Close) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hi"}}],"usage":{"completion_tokens":1}}`)) + })) + t.Cleanup(engine.Close) + + mir := &updMirror{base: files.URL, sums: map[string]string{ + "abc12345/q4.gguf": "aaaa", "def98765/q4.gguf": "bbbb"}} + // What was fetched before origins were kept: the mirror alone says where from. + _, _ = mir.Fetch(context.Background(), mirror.FetchRequest{Model: "qwen", Version: "abc12345", File: "q4.gguf", + Source: "https://huggingface.co/o/r/resolve/" + oldRev + "/q4.gguf"}) + models := &updModels{models: map[string]catalog.Model{"qwen": {Name: "qwen", + Source: files.URL + "/qwen/abc12345/q4.gguf", + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}}}, declared: map[string]bool{}} + reg := &movingRegistry{repo: hf.Repo{ID: "o/r", Revision: newRev, Files: []hf.File{ + {Path: "q4.gguf", SHA256: "bbbb", Bytes: 2 << 30}, {Path: "README.md", Bytes: 10}}}} + fleet := &restartFleet{engine: engine.URL, reps: []reconciler.Replica{ + {ID: "qwen-1", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}, + {ID: "qwen-2", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}}} + s := New(Deps{Registry: reg, Mirror: mir, Catalog: &fakeCatalog{}, Fleet: fleet, + Current: models.all, Models: models, Restart: fleet.Restart, + Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second}) + return s, mir, models, reg, fleet +} + +func TestNewWeightsInTheRepositoryAreFoundForAModelServedBeforeOriginsWereKept(t *testing.T) { + s, _, models, _, _ := updSetup(t) + s.CheckUpdates(context.Background()) + u := s.Updates()["qwen"] + if !u.Available || u.Repo != "o/r" || u.Current != oldRev || u.Latest != newRev || + len(u.Changed) != 1 || u.Changed[0] != "q4.gguf" || u.Bytes != 2<<30 { + t.Fatalf("%+v", u) + } + if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" || o.Files[0].SHA256 != "aaaa" { + t.Errorf("the origin worked out was not recorded: %+v", o) + } +} + +func TestANewCommitThatLeavesTheWeightsAloneIsNotAnUpdate(t *testing.T) { + s, mir, _, _, _ := updSetup(t) + mir.held[0].SHA256 = "bbbb" + u, err := s.CheckModel(context.Background(), "qwen") + if err != nil || u.Available || u.Latest != newRev || updateSays(u) != "same" { + t.Fatalf("%+v %v", u, err) + } +} + +func updateSays(u UpdateInfo) string { + if !u.Available && u.Latest != u.Current && len(u.Missing) == 0 { + return "same" + } + return "" +} + +func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) { + s, mir, models, _, fleet := updSetup(t) + oldSource := models.models["qwen"].Source + job, err := s.StartUpdate("qwen", "tester") + if err != nil { + t.Fatal(err) + } + j := wait(t, s, job.ID) + if j.State != Done { + t.Fatalf("%s: %+v", j.Error, j.Steps) + } + m := models.models["qwen"] + if !strings.HasSuffix(m.Source, "/qwen/def98765/q4.gguf") || m.Origin.Revision != newRev || + m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource { + t.Fatalf("catalog: %s %+v", m.Source, m.Origin) + } + if len(mir.held) != 2 { + t.Errorf("the old files did not stay on the mirror beside the new: %+v", mir.held) + } + for _, r := range fleet.Replicas() { + if !strings.Contains(r.ID, "new") { + t.Errorf("%s was not replaced", r.ID) + } + } + if u := s.Updates()["qwen"]; u.Available || u.Current != newRev { + t.Errorf("still offered: %+v", u) + } + + // And back. + if _, err := s.Rollback(context.Background(), "qwen", "tester"); err != nil { + t.Fatal(err) + } + m = models.models["qwen"] + if m.Source != oldSource || m.Origin.Revision != oldRev || m.Origin.Previous.Revision != newRev { + t.Fatalf("rollback: %s %+v", m.Source, m.Origin) + } + if u := s.Updates()["qwen"]; !u.Available { + t.Errorf("the update it went back from is not offered again: %+v", u) + } + + // And forward again, from files already on the mirror. + job, err = s.StartUpdate("qwen", "tester") + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done || models.models["qwen"].Origin.Revision != newRev { + t.Fatalf("%s: %+v", j.Error, j.Steps) + } + // Then there is nothing newer: refused, saying why. + job, _ = s.StartUpdate("qwen", "tester") + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") { + t.Fatalf("%+v", j) + } +} + +func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) { + s, _, models, _, _ := updSetup(t) + models.declared["qwen"] = true + if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") { + t.Fatalf("%v", err) + } +} + +func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) { + s, _, _, reg, _ := updSetup(t) + reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}} + job, _ := s.StartUpdate("qwen", "t") + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") { + t.Fatalf("%+v", j) + } +} + +func TestRegistryDownloadURLsAreRead(t *testing.T) { + for u, want := range map[string][3]string{ + "https://huggingface.co/bartowski/Qwen-GGUF/resolve/abc/sub/f.gguf": {"bartowski/Qwen-GGUF", "abc", "sub/f.gguf"}, + "https://hf.internal/mirror/x/resolve/main/f.gguf": {"mirror/x", "main", "f.gguf"}, + } { + repo, rev, path, ok := parseResolve(u) + if !ok || [3]string{repo, rev, path} != want { + t.Errorf("%s: %s %s %s %v", u, repo, rev, path, ok) + } + } + for _, u := range []string{"https://example.com/f.gguf", "https://huggingface.co/a/b/c/resolve/x/f", ":"} { + if _, _, _, ok := parseResolve(u); ok { + t.Errorf("%s was read", u) + } + } +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 303653f..b45be61 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -288,6 +288,38 @@ func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catal return final, err } +// SetSource moves a model added at runtime to other weights, recording where +// they came from. Running replicas keep what they loaded until they are +// replaced; the caller restarts them. +func (s *Service) SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its source", func(m *catalog.Model) error { + m.Source, m.Origin = source, origin + return nil + }) + if err == nil { + rev := "" + if origin != nil { + rev = origin.Revision + } + s.log.Info("model source set", "model", name, "source", source, "revision", rev, "by", by) + } + return final, err +} + +// SetOrigin records where a model's weights came from without changing them: +// for a model added before origins were kept. A model from configuration is +// left alone, its origin worked out again when it is needed. +func (s *Service) SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error { + if s.Declared(name) { + return nil + } + _, err := s.update(ctx, name, "llmgw", "record its origin", func(m *catalog.Model) error { + m.Origin = origin + return nil + }) + return err +} + // update changes one model added at runtime and keeps the change. // // Built before anything is written, like Add: catalog.New is what applies the diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 347f83b..408eb59 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -86,6 +86,7 @@ var Reference = []Block{ {Key: "tls", Type: "tlsconf.Config", Doc: "TLS authenticates to the mirror's management API, with the same CA the\nagents use. Empty leaves the connection in plain HTTP, which is only\nacceptable on a network where nothing untrusted can reach that port —\nthe mirror says as much in its own log when it starts that way.", Nested: "tlsconf.Config"}, {Key: "registry", Type: "string", Doc: "Registry is the model registry the mirror page browses, empty for the\npublic Hugging Face. Set it to an internal mirror of the registry where\nthe control plane cannot reach the internet — or leave the whole mirror\nblock out, which turns the explorer off rather than giving it a search\nbox that can only time out.", Nested: ""}, {Key: "timeout", Type: "time.Duration", Doc: "Timeout bounds a management call. It does not bound a download: the\nmirror answers a fetch as soon as it has started one, and the bytes keep\narriving long after this deadline has passed.", Nested: ""}, + {Key: "update_check", Type: "time.Duration", Doc: "UpdateCheck is how often the registry is asked whether the files of\nthe models served from it have changed there: a repository can publish\nnew weights under the same name. Only told, never applied — moving a\nmodel to them is `llmgw ctl model update`. 0 never asks.", Nested: ""}, }}, {Package: "catalog", Name: "Model", Title: "Model", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, @@ -104,6 +105,7 @@ var Reference = []Block{ {Key: "context_length", Type: "int", Doc: "ContextLength is how many tokens one request may use, prompt and\ncompletion together, and MaxOutput bounds the completion alone. Zero\nmeans unknown, which is what every model was until now.\n\nThe gateway knew neither, so an over-long prompt was not refused: it was\nadmitted, given a replica, and failed at the engine, and a client sizing\nits prompts from /v1/models had nothing to read. Worse, the number is not\nthe one an operator writes in the engine's arguments — llama.cpp divides\n--ctx-size across its parallel slots, so a pool of 8192 across four slots\nis 2048 a request, and nothing said so.", Nested: ""}, {Key: "max_output", Type: "int", Doc: "", Nested: ""}, {Key: "capabilities", Type: "Capabilities", Doc: "Capabilities is what this model can do beyond plain completion.", Nested: "catalog.Capabilities"}, + {Key: "origin", Type: "*Origin", Doc: "Origin is the registry repository the weights came from, at which\ncommit, and each file's checksum there. Written when a model is served\nfrom the registry; for one that was not, worked out from the mirror's\nrecord of where it fetched the file. It is what lets the control plane\nsay that the repository has published different weights since, and\nmove the model to them.", Nested: "catalog.Origin"}, }}, {Package: "config", Name: "Monitoring", Title: "Monitoring", Doc: "", Fields: []Field{ {Key: "prometheus", Type: "string", Doc: "Prometheus is the server's address, for example\nhttp://monitoring.internal:9090. Empty leaves the charts off the\ndashboard rather than showing an error where one should be.", Nested: ""}, @@ -116,6 +118,17 @@ var Reference = []Block{ {Key: "prefer_nvml", Type: "bool", Doc: "PreferNVML uses the NVML backend when this binary has one. NVLink\ntopology is read from nvidia-smi topo -m on every build; NVML, where it\nis used, is the more exact source of the peers.", Nested: ""}, {Key: "skip_gpus", Type: "bool", Doc: "SkipGPUs omits GPU discovery. A node without nvidia-smi is already\nfound to have none, so this is for one whose driver is broken.", Nested: ""}, }}, + {Package: "catalog", Name: "Origin", Title: "Origin", Doc: "", Fields: []Field{ + {Key: "repo", Type: "string", Doc: "Repo is the repository, owner/name.", Nested: ""}, + {Key: "revision", Type: "string", Doc: "Revision is the commit the files were fetched at.", Nested: ""}, + {Key: "files", Type: "[]OriginFile", Doc: "Files are the weights fetched from it, with what the registry\npublished as each one's sha256. A new commit that leaves them alone —\na README edited — is not an update.", Nested: ""}, + {Key: "previous", Type: "*PreviousOrigin", Doc: "Previous is what the last update replaced, kept so it can be gone back\nto while its files are still on the mirror.", Nested: "catalog.PreviousOrigin"}, + }}, + {Package: "catalog", Name: "OriginFile", Title: "OriginFile", Doc: "", Fields: []Field{ + {Key: "path", Type: "string", Doc: "", Nested: ""}, + {Key: "sha256", Type: "string", Doc: "", Nested: ""}, + {Key: "bytes", Type: "int64", Doc: "", Nested: ""}, + }}, {Package: "config", Name: "Personal", Title: "Personal", Doc: "", Fields: []Field{ {Key: "disabled", Type: "bool", Doc: "Disabled refuses personal keys and personal chat; people then use the\ngateway only through projects they are members of.", Nested: ""}, {Key: "rpm", Type: "int", Doc: "", Nested: ""}, @@ -132,6 +145,11 @@ var Reference = []Block{ {Key: "balance_abs", Type: "float64", Doc: "", Nested: ""}, {Key: "balance_rel", Type: "float64", Doc: "", Nested: ""}, }}, + {Package: "catalog", Name: "PreviousOrigin", Title: "PreviousOrigin", Doc: "", Fields: []Field{ + {Key: "source", Type: "string", Doc: "", Nested: ""}, + {Key: "revision", Type: "string", Doc: "", Nested: ""}, + {Key: "files", Type: "[]OriginFile", Doc: "", Nested: ""}, + }}, {Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, {Key: "engine", Type: "string", Doc: "", Nested: ""}, From dac7ff0e99d4aa85ee1088ba8675213653ce7690 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:19:50 +0200 Subject: [PATCH 2/5] The manual in the binary: llmgw docs, and Help in the console Every document and a page for every command, built from their help, read like info: topics, contents, find. ? opens the console's own page, which is new and covers every screen. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 32 ++ docs.go | 10 + docs/console.md | 200 ++++++++ internal/cli/root.go | 1 + internal/cli/tui_actions_test.go | 71 +++ internal/cli/tui_console.go | 54 ++- internal/cli/tui_docs.go | 764 +++++++++++++++++++++++++++++++ 7 files changed, 1129 insertions(+), 3 deletions(-) create mode 100644 docs.go create mode 100644 docs/console.md create mode 100644 internal/cli/tui_docs.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2444ca7..5509bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,38 @@ The people side runs the same commands as `llmgw users`, `keys` and `org`, against the same store, so their checks and messages are the ones shown. A sidebar taller than the terminal now scrolls with the current entry. +### New weights in a model's repository, found and moved to + +A repository on Hugging Face can publish new weights under the same file +names — a fixed chat template, a better quantisation — and nothing said so. A +model served from the registry now records its origin: the repository, the +commit, and each file's checksum there; a model served before is worked out +from the mirror's record of where it fetched each file. Every +`mirror.update_check` (six hours by default; 0 turns it off) the control +plane asks whether those files changed. A new commit that leaves them alone +is not an update. + +- `llmgw ctl model updates [--check]` lists what each repository has. +- `llmgw ctl model update MODEL` fetches the new files into the mirror beside + the old ones, checks them against the registry's checksums and the nodes' + llama.cpp, points the catalog at them, replaces the replicas one at a + time and asks one to say hello. +- `llmgw ctl model rollback MODEL` goes back to the files the update + replaced, which stay on the mirror. +- The console shows UPDATE on Models and offers all three on a model's row. + +A model from `server.yaml` is reported but not changed: its source is edited +there. + +### The manual, in the binary + +`llmgw docs` reads every document — installation, operation, the +configuration reference, the README, the changes — and a page for every +command, built from the commands' own help, like `info`: topics on the left, +T for a page's contents, / to find, n for the next match. Piped, it prints +the page. In the console it is **Help › Manual**, and **?** opens the new +page about the console itself from any screen. + ## v0.11.1 — 2026-10-03 ### The console's overview no longer spills over the screen diff --git a/docs.go b/docs.go new file mode 100644 index 0000000..11e35bc --- /dev/null +++ b/docs.go @@ -0,0 +1,10 @@ +// Package llmgw holds what the binary carries besides code: its manual. +package llmgw + +import "embed" + +// Docs is the documentation, read on any machine by `llmgw docs` and the +// console's manual without a checkout or the internet. +// +//go:embed README.md CHANGELOG.md docs/*.md +var Docs embed.FS diff --git a/docs/console.md b/docs/console.md new file mode 100644 index 0000000..676e27b --- /dev/null +++ b/docs/console.md @@ -0,0 +1,200 @@ +# The console + +`sudo llmgw`, with nothing after it, on a machine llmgw runs on. On a machine +with nothing installed it opens the setup screens; on a control plane, the +console described here; on a node, the node's own console, which edits +`agent.yaml`. Everything on these screens can also be done by command — each +screen says which — so a script never needs them and a person never needs +to remember the commands. + +The keyboard drives it; there is no mouse. **?** opens this page from +anywhere, and the **Manual** entry holds every other page, as `llmgw docs` +does without the console. + +## The layout + +A bar across the top names the machine. Down the left, the entries, in four +groups: **Watch**, **People**, **Help**, **Settings** and **Do**. The panel on +the right shows the entry chosen. The bottom line always says which keys do +what at that moment, and a question or a result opens as a dialog over the +panel. + +| Key | Does | +|---|---| +| ↑ ↓ | Choose an entry, or a row in the panel | +| → or Enter | Go into the panel | +| Enter on a row | What can be done to it: its details first, then the rest | +| ← or Esc | Back out: from a page to its list, from the panel to the entries | +| A | Add one, on a view that holds things: a quota, an account, a key | +| PgUp PgDn | Scroll a long panel | +| ? | This page | +| Q | Leave; the cluster keeps running | +| Ctrl-C | Leave at once | + +What is watched is read again every three seconds. + +## Forms and questions + +A change that needs values opens a form, filled in with what is set now. Tab +or ↓ moves to the next field, ← → changes a choice, and Enter on **Apply** +sends it; a form of one field is sent by Enter on it. A mistake the form can +see — a maximum below the minimum, a word where a number goes — is said in the +form and nothing is sent. One the control plane refuses comes back as a dialog +with its reason, word for word. + +Anything that cannot be undone — removing, draining, deleting, revoking — +asks first, and only **Y** goes ahead. + +Tasks that take a while and print as they go — serving a model, upgrading +the cluster, a backup, adding a node — close the screen and run in the +terminal, so their output is there to read; Enter brings the console back. + +## Watch + +**Overview** is `llmgw ctl status`: nodes, GPUs by class, fabrics, replicas, +models, the store's members, and anything parked, unplaced or idle. + +**Nodes** lists every node with its state, load, GPUs, replicas, disk and +agent version. On a row: + +- **Details** — its address, system, load, engines and GPUs, and any warning. +- **Cordon** places nothing new on it; **Uncordon** undoes it. +- **Drain** moves its replicas off one at a time, each replacement up before + the original stops. +- **vllm build / sglang build** (nodes with GPUs) — which build of the engine + the node installs: `auto` follows its hardware, `none` installs none. +- **Upgrade its agent** to a release; its engines keep serving. +- **Delete model files nothing uses** frees the node's disk. +- **Remove from the cluster** — the control plane forgets it; the machine is + not touched. + +**GPUs** is every card: how busy, its memory, temperature, power, and the +model holding it. + +**Models** lists the catalog: ready against wanted, status, class, profiles, +and **UPDATE** when the registry has new weights for it. On a row: + +- **Details** — description, replicas, class, owner, profiles, source, where + its weights came from, what it reserves against what it uses, and its + replicas. +- **Replicas (scale)** — the minimum kept running, the most it scales to, + and for a model loaded on demand how long it may sit unused. A maximum of + 0 stops it. +- **Class** — critical, standard or opportunistic: whose capacity gives way + when there is not enough for everyone. +- **Owner** — the organization, team or project whose capacity quota it + counts against. +- **Resources** — the memory and cores one replica reserves on its node. +- **LoRA adapters** — `name=source[@scale]`, separated by commas. +- **Restart its replicas**, one by one or in place. +- **Update to …** — when the repository it came from has published different + weights for its files: they are fetched into the mirror beside the ones in + use, checked, swapped in, and the replicas replaced one at a time. + **Go back to …** returns to the weights the update replaced. + **Ask the registry for new weights now** checks without waiting for the + next scheduled look (`mirror.update_check`, six hours by default). +- **Why is it placed where it is** — `llmgw ctl explain-placement`. +- **Try again**, for a model parked after failing every start. +- **Remove** — its replicas stop and it leaves the catalog. A model from + `server.yaml` is changed there instead; the console says so. + +**A** on Models adds one from a definition file on this machine: the YAML a +model takes in `server.yaml`. + +**Replicas** lists every replica; on one, **Restart it**, replaced or in +place. + +**Usage** is the last 24 hours by team, project or organization (→ changes +which): requests, tokens, errors and spend. + +**Quotas** are capacity quotas: the GPUs, cores and memory each organization, +team or project is guaranteed and may at most hold, and what it holds now. +**A** adds one; on a row, change or remove it. + +**Mirror** is what the model mirror holds, how far each download has got, and +which models use each file. **A** downloads a URL into it; on a row, where +nodes fetch it from, or remove it — with a warning when a model still uses it. + +**Logs** shows a node's agent log, newest at the bottom, errors and warnings +coloured. + +## People + +The accounts, keys and organizations live in the store; these views read it +and change it with the same commands as `llmgw users`, `llmgw keys` and +`llmgw org`, so their checks and messages are the ones shown. + +**Accounts** — who has signed up, their state, whether they see the cluster +dashboard, and what they belong to. **A** creates an account, already +approved. On a row: approve one waiting, suspend or reinstate, give or take +the cluster dashboard, add to or take out of an organization, team or +project, their limits, a new password (their sessions close), delete. + +**API keys** — every key, its project, owner, state and last use. **A** +issues one; the key is shown once, in the result, and never again. On a row: +its limits, or revoke it. + +**Organizations** — the tenancy tree: organizations, their teams, the teams' +projects, with members, keys and limits. **A** adds an organization. On a +row: add a team or project inside it, rename, move, limits for the level and +for each of its members, its capacity quota, and for a project how long +conversations and agent notes are kept. **Remove** takes away only something +empty. + +Limits are requests and tokens a minute, a budget per day or month, and the +models allowed. Each level only narrows what is above it. + +## Help + +**Manual** lists every page there is: these, the installation and operation +guides, the configuration reference, a page for every command, the README +and the changes in each release. On a page: + +| Key | Does | +|---|---| +| ↑ ↓, PgUp PgDn, Space | Scroll | +| g, G | The top, the end | +| T | Contents: the page's headings, Enter goes to one | +| / | Find; the lines holding it are marked | +| n, N | The next match, the one before | +| [ ] | The topic before, the one after | +| ← | Back to the list | + +`llmgw docs` opens the same manual outside the console, on any machine, +installed or not; `llmgw docs install` opens one page; piped, it prints the +page's text. + +## Settings + +**server.yaml** (on a node, **agent.yaml**) is every key of the file, as a +tree of menus built from the configuration reference: the value set, or the +one that applies when nothing is, its type, and what it is for. Enter changes +a value or opens a block; on a list or map of blocks, Enter opens an entry and +**D** removes one. **U** puts a key back to its default. **S** saves: the file +is checked the way the service reads it before anything is written, and its +comments are kept. The service reads it when it starts, so the console offers +to restart it. Another control plane keeps its own copy. + +## Do + +- **Add a node** — installs llmgw on a machine over ssh, checks it, gives it a + certificate and starts its agent (`sudo llmgw node add`). +- **Replicate the store** — moves the store onto two more machines under + Patroni (offered only while it is not replicated). +- **Serve a model** — Hugging Face's GGUF models: trending, most downloaded, + most liked or newest (**S** changes which), or **/** to search. Enter on + one lists its files with their sizes; the one chosen is fetched into the + mirror, checked, added to the catalog and started, and a replica is asked + to say hello (`llmgw ctl model serve`). `mirror.registry` points it at + another registry, for a site without the internet. +- **Upgrade the cluster** to a release: this machine, then every node in turn + (`sudo llmgw ctl upgrade`). +- **Restart all replicas**, one at a time. +- **Back up now** — the store and `/etc/llmgw` into one archive + (`sudo llmgw server backup`). +- **Leave**. + +## The node's console + +On a node, `sudo llmgw` opens `agent.yaml` in the same settings view, and the +manual. The cluster is watched and changed from a control plane. diff --git a/internal/cli/root.go b/internal/cli/root.go index 09e3f44..e6f61fd 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -58,6 +58,7 @@ func Root() *cobra.Command { newSelfUpgradeCmd(), newReleaseCmd(), newJoinCmd(), + newDocsCmd(), ) return root } diff --git a/internal/cli/tui_actions_test.go b/internal/cli/tui_actions_test.go index 1e5da5a..75e777e 100644 --- a/internal/cli/tui_actions_test.go +++ b/internal/cli/tui_actions_test.go @@ -319,3 +319,74 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { t.Fatalf("would run %q %q", act.kind, got) } } + +func TestTheManualHasEveryPageAndFindsInThem(t *testing.T) { + v := newDocViewer() + for _, k := range []string{"console", "install", "reference", "commands", "changes"} { + if findTopic(v.topics, k) < 0 { + t.Errorf("no %s page", k) + } + } + cmds := commandsManual() + for _, c := range []string{"llmgw ctl model update", "llmgw users approve", "llmgw docs"} { + if !strings.Contains(cmds, "# "+c) { + t.Errorf("the commands page lacks %s", c) + } + } + // Every line fits the width it was laid out for. + for _, tp := range v.topics { + for _, l := range renderMarkdown(tp.src(), 80) { + if !strings.HasPrefix(l.plain, " ") && len([]rune(l.plain)) > 80 { + t.Errorf("%s: a line of %d: %q", tp.key, len([]rune(l.plain)), l.plain) + break + } + } + } + v.show(findTopic(v.topics, "console")) + v.query = "drain" + v.find(80, 0, 1) + if l := v.lines(80)[v.scroll+2]; !strings.Contains(strings.ToLower(l.plain), "drain") { + t.Errorf("found at %q", l.plain) + } + v.openTOC(80) + if v.toc == nil || len(v.toc.items) < 5 { + t.Fatal("no contents") + } +} + +func TestQuestionMarkOpensTheConsolesPage(t *testing.T) { + m, _ := consoleWith(t, cNodes) + tm := press(m, "?") + cm := tm.(consoleModel) + if cm.menu != cManual || !cm.detail || cm.docs.topics[cm.docs.open].key != "console" { + t.Fatalf("menu %d detail %v", cm.menu, cm.detail) + } + if !strings.Contains(tm.View(), "The console") { + t.Error("the page is not shown") + } +} + +func TestNextFindsTheNextMatchAndComesRound(t *testing.T) { + v := newDocViewer() + v.show(findTopic(v.topics, "console")) + v.query = "revok" + lines := v.lines(80) + var hits []int + for i, l := range lines { + if strings.Contains(strings.ToLower(l.plain), "revok") { + hits = append(hits, i) + } + } + if len(hits) < 2 { + t.Fatalf("hits %v", hits) + } + v.find(80, 0, 1) + seen := []int{v.scroll + 2} + for range hits { + v.key(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("n")}, 80, 30) + seen = append(seen, v.scroll+2) + } + if seen[0] != hits[0] || seen[1] != hits[1] || seen[len(hits)] != hits[0] { + t.Errorf("hits %v, went %v", hits, seen) + } +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index ce7e888..4d59198 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -43,6 +43,7 @@ const ( cAccounts cKeys cOrgs + cManual cSettings cAddNode cReplicate @@ -54,7 +55,7 @@ const ( cCount ) -var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Logs", "Accounts", "API keys", "Organizations", +var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Logs", "Accounts", "API keys", "Organizations", "Manual", "server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster", "Restart all replicas", "Back up now", "Leave"} @@ -154,13 +155,14 @@ type consoleModel struct { people peopleData mirror consoleMirrorMsg hf hfState + docs docViewer } var usageGroupings = []string{"team", "project", "org"} func newConsoleModel(addr, file string) consoleModel { m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30, file: file, - st: &storeHolder{file: file}} + st: &storeHolder{file: file}, docs: newDocViewer()} m.node = strings.HasSuffix(file, "agent.yaml") if m.node { m.menu = cSettings @@ -329,7 +331,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m consoleModel) entryOK(i int) bool { if m.node { - return i == cSettings || i == cLeave + return i == cSettings || i == cManual || i == cLeave } return i != cReplicate || !m.storeReplicated() } @@ -346,6 +348,26 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { if mm, cmd, ok := m.actionKey(k); ok { return mm, cmd } + // The manual, from anywhere: its page about this screen. + if k.String() == "?" && !(m.menu == cManual && m.detail) { + m.menu, m.inPanel, m.detail = cManual, true, true + m.docs.show(max(0, findTopic(m.docs.topics, "console"))) + return m, nil + } + if m.menu == cManual && m.inPanel && m.detail { + pw := max(40, m.width-navWidth-8) + if m.docs.key(k, pw-4, m.height) { + return m, nil + } + switch k.String() { + case "esc", "left", "h", "backspace": + m.detail = false + m.row = m.docs.open + case "q": + return m.leave() + } + return m, nil + } switch k.Type { case tea.KeyPgDown: m.scroll += max(1, m.height-8) @@ -434,6 +456,9 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.menuD = m.keyMenu(m.people.keys[m.row]) case cOrgs: m.menuD = m.orgMenu(m.people.orgs[m.row]) + case cManual: + m.docs.show(m.row) + m.detail = true } case "/": if m.menu == cServe { @@ -808,6 +833,8 @@ func (m consoleModel) rowCount() int { return len(m.people.keys) case cOrgs: return len(m.people.orgs) + case cManual: + return len(m.docs.topics) } return 0 } @@ -825,6 +852,8 @@ func (m consoleModel) View() string { switch i { case cAccounts: nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "People", Heading: true}) + case cManual: + nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Help", Heading: true}) case cSettings: nav = append(nav, navItem{Label: "", Heading: true}, navItem{Label: "Settings", Heading: true}) case cAddNode, cLeave: @@ -850,6 +879,9 @@ func (m consoleModel) View() string { Width: m.width, Height: m.height, Title: title0, Where: host, NavTitle: "", Nav: nav, PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys, } + if m.menu == cManual && m.detail { + f.Scroll = m.docs.scroll + } // The selected row kept on screen. if m.inPanel && !m.detail { inner := m.height - 4 @@ -869,6 +901,8 @@ func (m consoleModel) View() string { } f.Dialog = q + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel") } + } else if t, d := m.docs.dialog(); d != "" && m.menu == cManual { + f.DialogTitle, f.Dialog = t, d } else if m.busy != "" { f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…" } else if t, d := m.actionDialog(); d != "" { @@ -1004,6 +1038,20 @@ func (m consoleModel) panel() (title, body, keys string) { return m.quotasPanel(pw) case cAccounts, cKeys, cOrgs: return m.peoplePanel(pw) + case cManual: + if m.detail && m.docs.open >= 0 { + k := docKeys + if m.docs.note != "" { + k = m.docs.note + } + return "Manual · " + m.docs.topics[m.docs.open].title, "\n" + m.docs.body(pw-4), k + } + d := m.docs + d.sel = m.row + if m.inPanel { + keys = "↑↓ topic · Enter read · ← back · ? the console's page · Q leave" + } + return "Manual", d.list(pw, m.inPanel), keys case cMirror: return m.mirrorPanel(pw) case cServe: diff --git a/internal/cli/tui_docs.go b/internal/cli/tui_docs.go new file mode 100644 index 0000000..ee689bf --- /dev/null +++ b/internal/cli/tui_docs.go @@ -0,0 +1,764 @@ +package cli + +import ( + "fmt" + "io/fs" + "sort" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/spf13/cobra" + + llmgw "gitea.mynodes.xyz/ilker/llmgw" +) + +// The manual: every document the project has, carried in the binary, and a +// page for every command built from the commands themselves — read like man +// or info, in the console or with `llmgw docs`, on a machine with no checkout +// and no browser. + +type docTopic struct { + key string // what `llmgw docs KEY` takes + title string + src func() string +} + +// docOrder is the documents in the order someone new reads them. +var docOrder = []string{"console", "quickstart", "quick-install", "install", "node-onboarding", "onboarding", + "choosing-models", "fabrics", "facility-aware-placement", "data", "reference"} + +func docTopics() []docTopic { + read := func(path string) func() string { + return func() string { + b, err := fs.ReadFile(llmgw.Docs, path) + if err != nil { + return "# " + path + "\n\nNot in this build." + } + return string(b) + } + } + titleOf := func(path string) string { + b, _ := fs.ReadFile(llmgw.Docs, path) + for _, l := range strings.Split(string(b), "\n") { + if strings.HasPrefix(l, "# ") { + return strings.TrimPrefix(l, "# ") + } + } + return path + } + var out []docTopic + seen := map[string]bool{} + add := func(key, path string) { + if seen[key] { + return + } + if _, err := fs.Stat(llmgw.Docs, path); err != nil { + return + } + seen[key] = true + out = append(out, docTopic{key: key, title: titleOf(path), src: read(path)}) + } + for _, k := range docOrder { + add(k, "docs/"+k+".md") + if k == "reference" { + out = append(out, docTopic{key: "commands", title: "Commands", src: commandsManual}) + seen["commands"] = true + } + } + // Any document not placed above, so a new one is never missing. + entries, _ := fs.ReadDir(llmgw.Docs, "docs") + var rest []string + for _, e := range entries { + rest = append(rest, strings.TrimSuffix(e.Name(), ".md")) + } + sort.Strings(rest) + for _, k := range rest { + add(k, "docs/"+k+".md") + } + add("readme", "README.md") + add("changes", "CHANGELOG.md") + return out +} + +func findTopic(topics []docTopic, key string) int { + key = strings.ToLower(strings.TrimSuffix(key, ".md")) + for i, t := range topics { + if t.key == key { + return i + } + } + for i, t := range topics { + if strings.Contains(strings.ToLower(t.title), key) || strings.HasPrefix(t.key, key) { + return i + } + } + return -1 +} + +// commandsManual is a page for every command, from its own help. +func commandsManual() string { + var b strings.Builder + b.WriteString("# Commands\n\nEvery command, as `llmgw COMMAND --help` describes it. " + + "Run bare, in a terminal, `llmgw` opens the setup screens or the console instead.\n") + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + if c.Hidden || c.Name() == "help" || c.Name() == "completion" { + return + } + if c.HasParent() { + level := min(strings.Count(c.CommandPath(), " ")+1, 4) + fmt.Fprintf(&b, "\n%s %s\n\n", strings.Repeat("#", level), c.CommandPath()) + if c.Long != "" { + b.WriteString(c.Long + "\n") + } else if c.Short != "" { + b.WriteString(c.Short + ".\n") + } + b.WriteString("\n```\n" + c.UseLine() + "\n") + if c.Example != "" { + b.WriteString("\n" + strings.TrimRight(c.Example, "\n") + "\n") + } + if fl := c.LocalFlags(); fl.HasAvailableFlags() { + b.WriteString("\n" + strings.TrimRight(fl.FlagUsagesWrapped(100), "\n") + "\n") + } + b.WriteString("```\n") + } + subs := c.Commands() + sort.Slice(subs, func(i, j int) bool { return subs[i].Name() < subs[j].Name() }) + for _, s := range subs { + walk(s) + } + } + walk(Root()) + return b.String() +} + +// --- markdown, for a terminal --- + +var ( + mdH1 = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent).Underline(true) + mdH2 = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent) + mdH3 = lipgloss.NewStyle().Bold(true) + mdCode = lipgloss.NewStyle().Foreground(lipgloss.AdaptiveColor{Light: "#8A4B08", Dark: "#E5C07B"}) + mdBold = lipgloss.NewStyle().Bold(true) + mdHit = lipgloss.NewStyle().Reverse(true) + mdEm = lipgloss.NewStyle().Italic(true) + mdLink = lipgloss.NewStyle().Underline(true) +) + +// docLine is one line of a rendered document: as shown, as plain text for +// searching, and its heading level if it is one. +type docLine struct { + text, plain string + heading int +} + +type mdRun struct { + text string + style *lipgloss.Style + link string +} + +// inline splits markdown's inline markup into styled runs. +func inline(s string) []mdRun { + var out []mdRun + plain := func(t string) { + if t != "" { + out = append(out, mdRun{text: t}) + } + } + for s != "" { + i := strings.IndexAny(s, "`*[") + if i < 0 { + plain(s) + break + } + plain(s[:i]) + s = s[i:] + switch { + case s[0] == '`': + if j := strings.IndexByte(s[1:], '`'); j >= 0 { + out = append(out, mdRun{text: s[1 : j+1], style: &mdCode}) + s = s[j+2:] + continue + } + case strings.HasPrefix(s, "**"): + if j := strings.Index(s[2:], "**"); j >= 0 { + out = append(out, mdRun{text: s[2 : j+2], style: &mdBold}) + s = s[j+4:] + continue + } + case s[0] == '*': + // *emphasis*, when it closes and is not a lone star in text. + if j := strings.IndexByte(s[1:], '*'); j > 0 && s[1] != ' ' { + out = append(out, mdRun{text: s[1 : j+1], style: &mdEm}) + s = s[j+2:] + continue + } + case s[0] == '[': + if j := strings.Index(s, "]("); j > 0 { + if k := strings.IndexByte(s[j:], ')'); k > 0 { + out = append(out, mdRun{text: s[1:j], link: s[j+2 : j+k]}) + s = s[j+k+1:] + continue + } + } + } + plain(s[:1]) + s = s[1:] + } + return out +} + +// wrapRuns lays styled runs out in lines of at most width columns: words +// are cut at spaces, a word keeping each part's own style. +func wrapRuns(runs []mdRun, width int, first, rest string) []docLine { + type word struct{ plain, shown string } + var words []word + var plain, shown strings.Builder + end := func() { + if plain.Len() > 0 { + words = append(words, word{plain.String(), shown.String()}) + } + plain.Reset() + shown.Reset() + } + for _, r := range runs { + text := r.text + if r.link != "" && strings.HasPrefix(r.link, "http") { + text += " ‹" + r.link + "›" + } + for k, part := range strings.Split(text, " ") { + if k > 0 { + end() + } + if part == "" { + continue + } + plain.WriteString(part) + switch { + case r.style != nil: + shown.WriteString(r.style.Render(part)) + case r.link != "": + shown.WriteString(mdLink.Render(part)) + default: + shown.WriteString(part) + } + } + } + end() + + var out []docLine + var line, lp strings.Builder + start := func(pre string) { + line.Reset() + lp.Reset() + line.WriteString(pre) + lp.WriteString(pre) + } + start(first) + col, empty := len([]rune(first)), true + for _, w := range words { + n := len([]rune(w.plain)) + if !empty && col+1+n > width { + out = append(out, docLine{text: line.String(), plain: lp.String()}) + start(rest) + col, empty = len([]rune(rest)), true + } + if !empty { + line.WriteString(" ") + lp.WriteString(" ") + col++ + } + line.WriteString(w.shown) + lp.WriteString(w.plain) + col += n + empty = false + } + if !empty { + out = append(out, docLine{text: line.String(), plain: lp.String()}) + } + return out +} + +// renderMarkdown lays a document out for a terminal width. +func renderMarkdown(src string, width int) []docLine { + width = max(width, 30) + var out []docLine + blank := func() { + if len(out) > 0 && out[len(out)-1].plain != "" { + out = append(out, docLine{}) + } + } + lines := strings.Split(strings.ReplaceAll(src, "\r\n", "\n"), "\n") + var para []string + flush := func() { + if len(para) > 0 { + out = append(out, wrapRuns(inline(strings.Join(para, " ")), width, "", "")...) + para = nil + } + } + for i := 0; i < len(lines); i++ { + l := lines[i] + t := strings.TrimSpace(l) + switch { + case strings.HasPrefix(t, "```"): + flush() + blank() + for i++; i < len(lines) && !strings.HasPrefix(strings.TrimSpace(lines[i]), "```"); i++ { + code := " " + strings.ReplaceAll(lines[i], "\t", " ") + out = append(out, docLine{text: mdCode.Render(truncateWidth(code, width)), plain: code}) + } + blank() + case t == "": + flush() + blank() + case strings.HasPrefix(t, "#"): + flush() + level := len(t) - len(strings.TrimLeft(t, "#")) + title := strings.TrimSpace(t[level:]) + plainTitle := "" + for _, r := range inline(title) { + plainTitle += r.text + } + style := mdH3 + switch level { + case 1: + style = mdH1 + case 2: + style = mdH2 + } + blank() + out = append(out, docLine{text: style.Render(plainTitle), plain: plainTitle, heading: level}) + out = append(out, docLine{}) + case strings.HasPrefix(t, "|"): + flush() + var rows [][]string + for ; i < len(lines) && strings.HasPrefix(strings.TrimSpace(lines[i]), "|"); i++ { + cells := strings.Split(strings.Trim(strings.TrimSpace(lines[i]), "|"), "|") + sep := true + for j := range cells { + cells[j] = strings.TrimSpace(cells[j]) + if strings.Trim(cells[j], ":-") != "" { + sep = false + } + } + if !sep { + rows = append(rows, cells) + } + } + i-- + out = append(out, renderTable(rows, width)...) + case strings.HasPrefix(t, ">"): + flush() + q := strings.TrimSpace(strings.TrimPrefix(t, ">")) + for _, d := range wrapRuns(inline(q), width-2, "", "") { + out = append(out, docLine{text: sDim.Render("│ ") + d.text, plain: "│ " + d.plain}) + } + case listItem(t) != "": + flush() + marker := listItem(t) + indent := strings.Repeat(" ", len(l)-len(strings.TrimLeft(l, " "))) + body := strings.TrimSpace(t[len(marker):]) + // The item's following lines, indented under it. + for i+1 < len(lines) { + nt := strings.TrimSpace(lines[i+1]) + if nt == "" || listItem(nt) != "" || strings.HasPrefix(nt, "```") || strings.HasPrefix(nt, "#") || + !strings.HasPrefix(lines[i+1], " ") { + break + } + body += " " + nt + i++ + } + bullet := "• " + if marker != "- " && marker != "* " { + bullet = marker + } + out = append(out, wrapRuns(inline(body), width, indent+bullet, indent+strings.Repeat(" ", len([]rune(bullet))))...) + default: + para = append(para, t) + } + } + flush() + return out +} + +func listItem(t string) string { + if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") { + return t[:2] + } + for i, c := range t { + if c >= '0' && c <= '9' { + continue + } + if i > 0 && i < 4 && c == '.' && strings.HasPrefix(t[i+1:], " ") { + return t[:i+2] + } + break + } + return "" +} + +// renderTable aligns a markdown table, wrapping the widest column to fit. +func renderTable(rows [][]string, width int) []docLine { + if len(rows) == 0 { + return nil + } + plainCell := func(s string) string { + var p string + for _, r := range inline(s) { + p += r.text + } + return p + } + n := 0 + for _, r := range rows { + n = max(n, len(r)) + } + w := make([]int, n) + for _, r := range rows { + for j, c := range r { + w[j] = max(w[j], len([]rune(plainCell(c)))) + } + } + total := func() int { + t := 0 + for _, x := range w { + t += x + 2 + } + return t + } + for total() > width { + widest := 0 + for j := range w { + if w[j] > w[widest] { + widest = j + } + } + if w[widest] <= 12 { + break + } + w[widest]-- + } + var out []docLine + for ri, r := range rows { + cols := make([][]string, n) + height := 1 + for j := 0; j < n; j++ { + c := "" + if j < len(r) { + c = plainCell(r[j]) + } + cols[j] = strings.Split(wrap(c, max(w[j], 20), ""), "\n") + if len([]rune(c)) <= w[j] { + cols[j] = []string{c} + } + height = max(height, len(cols[j])) + } + for h := 0; h < height; h++ { + var b strings.Builder + for j := 0; j < n; j++ { + c := "" + if h < len(cols[j]) { + c = cols[j][h] + } + b.WriteString(fmt.Sprintf("%-*s ", w[j], truncate(c, w[j]))) + } + plain := strings.TrimRight(b.String(), " ") + text := plain + if ri == 0 { + text = mdBold.Render(plain) + } + out = append(out, docLine{text: text, plain: plain}) + } + } + return out +} + +// --- the viewer --- + +// docViewer is the manual: a list of topics, one open, its contents, and a +// search. Both the console and `llmgw docs` show it. +type docViewer struct { + topics []docTopic + open int // the topic shown; -1 is the list + sel int // the topic chosen in the list + scroll int + query string + // toc and search are dialogs over the page. + toc *menuDialog + tocLines []int + asking bool + prompt textField + note string +} + +func newDocViewer() docViewer { return docViewer{topics: docTopics(), open: -1} } + +func (v *docViewer) lines(width int) []docLine { + if v.open < 0 { + return nil + } + return renderMarkdown(v.topics[v.open].src(), width) +} + +// key handles a key on an open page; false hands it back. +func (v *docViewer) key(k tea.KeyMsg, width, height int) bool { + if v.asking { + switch k.Type { + case tea.KeyEsc: + v.asking = false + case tea.KeyEnter: + v.asking = false + v.query = strings.TrimSpace(v.prompt.Value()) + v.find(width, v.scroll, 1) + default: + v.prompt, _ = v.prompt.Update(k) + } + return true + } + if v.toc != nil { + switch k.String() { + case "up", "k": + v.toc.sel = (v.toc.sel - 1 + len(v.toc.items)) % len(v.toc.items) + case "down", "j", "tab": + v.toc.sel = (v.toc.sel + 1) % len(v.toc.items) + case "pgdown": + v.toc.sel = min(len(v.toc.items)-1, v.toc.sel+10) + case "pgup": + v.toc.sel = max(0, v.toc.sel-10) + case "enter": + v.scroll = max(0, v.tocLines[v.toc.sel]-1) + v.toc = nil + case "esc", "q", "t": + v.toc = nil + } + return true + } + if v.open < 0 { + return false + } + n := len(v.lines(width)) + page := max(1, height-6) + switch k.String() { + case "up", "k": + v.scroll-- + case "down", "j", "enter": + v.scroll++ + case "pgdown", " ", "f": + v.scroll += page + case "pgup", "b": + v.scroll -= page + case "home", "g": + v.scroll = 0 + case "end", "G": + v.scroll = n - page + case "t": + v.openTOC(width) + case "/": + v.asking = true + v.prompt = newTextField() + v.prompt.Width = 30 + v.prompt.SetValue(v.query) + v.prompt.Focus() + case "n": + // A match is shown two lines down; the next is after it. + v.find(width, v.scroll+3, 1) + case "N": + v.find(width, v.scroll+1, -1) + case "]": + v.show(min(len(v.topics)-1, v.open+1)) + case "[": + v.show(max(0, v.open-1)) + default: + return false + } + v.scroll = max(0, min(v.scroll, n-1)) + return true +} + +func (v *docViewer) show(i int) { + v.open, v.sel, v.scroll, v.note = i, i, 0, "" +} + +func (v *docViewer) openTOC(width int) { + v.toc, v.tocLines = &menuDialog{title: "Contents · " + v.topics[v.open].title}, nil + for i, l := range v.lines(width) { + if l.heading == 0 || l.heading > 3 { + continue + } + v.toc.items = append(v.toc.items, menuItem{label: strings.Repeat(" ", max(0, l.heading-1)) + truncate(l.plain, 60)}) + v.tocLines = append(v.tocLines, i) + if i <= v.scroll+1 { + v.toc.sel = len(v.tocLines) - 1 + } + } + if len(v.toc.items) == 0 { + v.toc = nil + } +} + +// find moves to the next line holding the query, from line `from` in +// direction dir, wrapping round. +func (v *docViewer) find(width, from, dir int) { + if v.query == "" { + return + } + lines := v.lines(width) + q := strings.ToLower(v.query) + for k := 0; k < len(lines); k++ { + i := ((from+k*dir)%len(lines) + len(lines)) % len(lines) + if strings.Contains(strings.ToLower(lines[i].plain), q) { + v.scroll = max(0, i-2) + v.note = "" + return + } + } + v.note = "“" + v.query + "” is not in this page" +} + +// body is the open page, the query's lines marked. +func (v *docViewer) body(width int) string { + var b strings.Builder + q := strings.ToLower(v.query) + for _, l := range v.lines(width) { + text := l.text + if q != "" && strings.Contains(strings.ToLower(l.plain), q) { + text = mdHit.Render(" ") + " " + text + } else { + text = " " + text + } + b.WriteString(text + "\n") + } + return b.String() +} + +// list is the topics, as a table. +func (v *docViewer) list(width int, focused bool) string { + var rows [][]string + for _, t := range v.topics { + rows = append(rows, []string{t.title, sDim.Render("llmgw docs " + t.key)}) + } + return "\n " + sDim.Render("Everything there is to read about llmgw, carried in the binary.") + "\n\n" + + table([]string{"TOPIC", "ALSO"}, rows, v.sel, width, focused) +} + +// dialog is the contents or the search, open over the page. +func (v *docViewer) dialog() (string, string) { + switch { + case v.asking: + return "Find in this page", v.prompt.View() + "\n\n" + sDim.Render("Enter · then n next, N back · Esc cancel") + case v.toc != nil: + d := consoleModel{menuD: v.toc, height: 40} + _, body := d.actionDialog() + return v.toc.title, body + case v.note != "": + return "", "" + } + return "", "" +} + +const docKeys = "↑↓ PgUp/PgDn g/G scroll · T contents · / find · n/N next · [ ] topic · ← back" + +// --- `llmgw docs` --- + +type docsModel struct { + v docViewer + width, height int + inPage bool +} + +func (m docsModel) Init() tea.Cmd { return nil } + +func (m docsModel) pageWidth() int { return max(30, m.width-navWidth-10) } + +func (m docsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + case tea.KeyMsg: + if msg.Type == tea.KeyCtrlC { + return m, tea.Quit + } + if m.inPage && m.v.key(msg, m.pageWidth(), m.height) { + return m, nil + } + switch msg.String() { + case "q": + return m, tea.Quit + case "up", "k": + if !m.inPage { + m.v.sel = (m.v.sel - 1 + len(m.v.topics)) % len(m.v.topics) + m.v.show(m.v.sel) + } + case "down", "j": + if !m.inPage { + m.v.sel = (m.v.sel + 1) % len(m.v.topics) + m.v.show(m.v.sel) + } + case "enter", "right", "l", "tab": + m.inPage = true + case "esc", "left", "h", "backspace": + if m.inPage { + m.inPage = false + } else { + return m, tea.Quit + } + } + } + return m, nil +} + +func (m docsModel) View() string { + var nav []navItem + for i, t := range m.v.topics { + nav = append(nav, navItem{Label: t.title, Current: i == m.v.open}) + } + keys := "↑↓ topic · → or Enter read · Q leave" + if m.inPage { + keys = docKeys + } + if m.v.note != "" { + keys = m.v.note + } + f := frame{Width: m.width, Height: m.height, Title: "llmgw manual", Where: "llmgw docs", + Nav: nav, PanelTitle: m.v.topics[m.v.open].title, Body: "\n" + m.v.body(m.pageWidth()), + Scroll: m.v.scroll, Keys: keys} + f.DialogTitle, f.Dialog = m.v.dialog() + return f.render() +} + +func newDocsCmd() *cobra.Command { + return &cobra.Command{ + Use: "docs [TOPIC]", + Short: "Read the manual: every document, and a page for every command", + Long: "In a terminal, a reader like info: the topics down the left, the page on\n" + + "the right, T for its contents, / to find. Piped, the topic's text, or the\n" + + "list of topics. TOPIC is a name from that list or a word of its title.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + v := newDocViewer() + i := 0 + if len(args) == 1 { + if i = findTopic(v.topics, args[0]); i < 0 { + return fmt.Errorf("no topic %q; `llmgw docs` lists them", args[0]) + } + } + out := cmd.OutOrStdout() + if !interactive(cmd) { + if len(args) == 0 { + for _, t := range v.topics { + fmt.Fprintf(out, "%-26s %s\n", t.key, t.title) + } + return nil + } + _, err := fmt.Fprint(out, v.topics[i].src()) + return err + } + v.show(i) + m := docsModel{v: v, width: 100, height: 30, inPage: len(args) == 1} + _, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run() + return err + }, + } +} From f36c46a8813d56d7618b600dd03870458dc77547 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:24:41 +0200 Subject: [PATCH 3/5] Trace weights fetched before origins were kept, and let an origin be given by hand The mirror keeps where it fetched a file beside it, and reports that and the checksum after it restarts. A model whose files the mirror cannot trace is found on the registry: a repository holding a file of that name that has the commit the mirror's directory is named after, with every file at the same size, the most downloaded first. ctl model origin and the console set one by hand, checked the same way. Commits compare shortened. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/cli/model.go | 2 +- internal/cli/model_update.go | 22 +++ internal/cli/tui_actions.go | 27 ++++ internal/control/adminapi/api.go | 24 +++ internal/control/modelserve/modelserve.go | 5 +- internal/control/modelserve/updates.go | 155 ++++++++++++++++++-- internal/control/modelserve/updates_test.go | 76 ++++++++++ internal/hf/hf.go | 24 ++- internal/mirror/add.go | 5 + internal/mirror/manager.go | 5 + internal/mirror/mirror.go | 25 ++++ 11 files changed, 356 insertions(+), 14 deletions(-) diff --git a/internal/cli/model.go b/internal/cli/model.go index 94e4f48..66c6d28 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -36,7 +36,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), - newModelRollbackCmd(client)) + newModelRollbackCmd(client), newModelOriginCmd(client)) return c } diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go index 4111663..fe985dd 100644 --- a/internal/cli/model_update.go +++ b/internal/cli/model_update.go @@ -117,3 +117,25 @@ func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { }, } } + +func newModelOriginCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "origin MODEL OWNER/REPO", + Short: "Say which registry repository a model's weights came from", + Long: "For a model fetched before llmgw kept this, and that it could not find on\n" + + "its own. The repository is checked: it must have the commit the mirror's\n" + + "directory is named after, holding the model's files at the sizes the\n" + + "mirror has. From then on its updates are watched.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + body, _ := json.Marshal(map[string]string{"repo": args[1]}) + var o struct{ Repo, Revision string } + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/origin", body, &o); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s came from %s at %s; its updates are watched\n", args[0], o.Repo, + catalog.ShortRevision(o.Revision)) + return nil + }, + } +} diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 05a0784..a4743c8 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -680,7 +680,33 @@ func (m *consoleModel) doEntry() tea.Cmd { // updateItems are what can be done about a model's repository publishing new // weights: move to them, ask now, or go back. func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { + origin := menuItem{"Where its weights came from…", func(m *consoleModel) tea.Cmd { + repo := "" + if md.Origin != nil { + repo = md.Origin.Repo + } + m.openForm(&formDialog{title: "Origin of " + md.Name, + note: "The registry repository its files were fetched from. Checked: it must have the\n" + + "commit the mirror's directory is named after, with the same files at the same sizes.", + fields: []formField{textF("repository", repo, "owner/name, like bartowski/phi-4-GGUF")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if !strings.Contains(v[0], "/") { + return nil, fmt.Errorf("owner/name") + } + return m.call("Origin of "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/origin", + map[string]string{"repo": v[0]}, func(raw json.RawMessage) string { + var o adminapi.OriginView + _ = json.Unmarshal(raw, &o) + return fmt.Sprintf("%s came from %s at %s; its updates are watched.", md.Name, o.Repo, + catalog.ShortRevision(o.Revision)) + }), nil + }}) + return nil + }} if md.Origin == nil { + if strings.Contains(md.Source, "://") { + return []menuItem{origin} + } return nil } server := "--server=" + m.client.base @@ -712,6 +738,7 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u)) }) }}) + items = append(items, origin) if prev := md.Origin.Previous; prev != "" { items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd { m.confirmD = &confirmDialog{title: "Go back on " + md.Name, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a32d851..fbf4db4 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -125,6 +125,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) + mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -1520,6 +1521,29 @@ func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"result": msg}) } +// OriginRequest names the repository a model's weights came from. +type OriginRequest struct { + Repo string `json:"repo"` +} + +func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + var req OriginRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil || req.Repo == "" { + writeError(w, http.StatusBadRequest, "name the repository, owner/name") + return + } + o, err := a.serve.SetOriginOf(r.Context(), r.PathValue("name"), req.Repo) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision}) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 3ef4807..d68491d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -146,6 +146,9 @@ type Server struct { seq int // updates is what the last checks of the registry found, by model. updates map[string]UpdateInfo + // origins are those worked out for models whose definition does not + // keep one, nil for a model that could not be traced. + origins map[string]*catalog.Origin } // New builds a server. @@ -162,7 +165,7 @@ func New(d Deps) *Server { if d.Log == nil { d.Log = slog.Default() } - return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}} + return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}, origins: map[string]*catalog.Origin{}} } var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index cc446e8..62bebb6 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "regexp" "slices" "strings" "sync/atomic" @@ -124,35 +125,156 @@ func originFiles(set hf.FileSet) []catalog.OriginFile { return out } -// originOf is a model's origin: recorded, or worked out from the mirror and -// then recorded. +// Finder is a registry that can be searched, and read at a past commit: what +// finding the origin of weights fetched before origins were kept needs. +type Finder interface { + Explore(ctx context.Context, q hf.Query) ([]hf.Model, error) + RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) +} + +// originOf is a model's origin: recorded; or worked out from where the +// mirror fetched its files; or found on the registry by the files' names, +// the commit the mirror's directory is named after, and their sizes. What is +// worked out is recorded, so it is worked out once. func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin { if m.Origin != nil && m.Origin.Repo != "" { return m.Origin } + s.mu.Lock() + o, known := s.origins[m.Name] + s.mu.Unlock() + if known { + return o + } p, ok := placeOf(m.Source, list) if !ok { return nil } - o, ok := inferOrigin(p) + o, ok = inferOrigin(p) if !ok { + o = s.findOrigin(ctx, p) + } + s.remember(ctx, m.Name, o) + return o +} + +// remember keeps an origin: in the catalog for a model added at runtime, and +// here for one from configuration, whose definition is not this one's to +// change. A nil origin is remembered too, so a model nobody can trace is not +// searched for at every check. +func (s *Server) remember(ctx context.Context, name string, o *catalog.Origin) { + s.mu.Lock() + s.origins[name] = o + s.mu.Unlock() + if o == nil || s.deps.Models == nil || s.deps.Models.Declared(name) { + return + } + if err := s.deps.Models.SetOrigin(ctx, name, o); err != nil { + s.deps.Log.Debug("could not record a model's origin", "model", name, "err", err) + } +} + +var hexRev = regexp.MustCompile(`^[0-9a-f]{7,40}$`) + +// findOrigin looks for the repository a mirror's files came from: one that +// holds a file of the same name, and has the commit the mirror's directory +// is named after with every file at the same size. Searched by the mirror's +// name for the model and the file's own; among the repositories that match, +// the most downloaded, which is the original rather than a copy of it. +func (s *Server) findOrigin(ctx context.Context, p place) *catalog.Origin { + f, ok := s.deps.Registry.(Finder) + if !ok || !hexRev.MatchString(p.version) { return nil } - if s.deps.Models != nil { - if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil { - s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err) + file := p.files[0].File + stem := strings.TrimSuffix(file, ".gguf") + if i := strings.LastIndexAny(stem, "-."); i > 0 { + stem = stem[:i] // the quantisation + } + tried := map[string]bool{} + for _, q := range []string{p.model, stem} { + found, err := f.Explore(ctx, hf.Query{Search: q, GGUF: true, Sort: hf.SortDownloads, Limit: 100, WithFiles: true}) + if err != nil { + s.deps.Log.Debug("searching the registry for a model's origin", "model", p.model, "err", err) + continue + } + for _, c := range found { + if tried[c.ID] || !slices.ContainsFunc(c.Siblings, func(sb struct { + RFilename string `json:"rfilename"` + }) bool { + return mirror.LastPathElement(sb.RFilename) == file + }) { + continue + } + tried[c.ID] = true + if o := s.verifyOrigin(ctx, f, c.ID, p); o != nil { + s.deps.Log.Info("found where a model's weights came from", "model", p.model, "repo", c.ID, + "revision", catalog.ShortRevision(o.Revision)) + return o + } } } + return nil +} + +// verifyOrigin reads a repository at the mirror's commit and says whether +// every file the mirror holds is there at the same size. +func (s *Server) verifyOrigin(ctx context.Context, f Finder, id string, p place) *catalog.Origin { + repo, err := f.RepoAt(ctx, id, p.version) + if err != nil || !strings.HasPrefix(repo.Revision, p.version) { + return nil + } + o := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision} + for _, st := range p.files { + i := slices.IndexFunc(repo.Files, func(g hf.File) bool { + return mirror.LastPathElement(g.Path) == st.File && g.Bytes == st.BytesDone + }) + if i < 0 { + return nil + } + g := repo.Files[i] + o.Files = append(o.Files, catalog.OriginFile{Path: g.Path, SHA256: g.SHA256, Bytes: g.Bytes}) + } return o } +// SetOriginOf records, by hand, the repository a model's weights came from; +// the commit is the one the mirror's directory is named after, and the files +// must be there at the sizes the mirror holds. +func (s *Server) SetOriginOf(ctx context.Context, name, repoID string) (*catalog.Origin, error) { + m, err := s.current(name) + if err != nil { + return nil, err + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return nil, fmt.Errorf("the mirror: %w", err) + } + p, ok := placeOf(m.Source, list) + if !ok { + return nil, fmt.Errorf("%s does not run from the mirror", name) + } + f, ok := s.deps.Registry.(Finder) + if !ok { + return nil, errors.New("this registry cannot be read at a past commit") + } + o := s.verifyOrigin(ctx, f, repoID, p) + if o == nil { + return nil, fmt.Errorf("%s has no commit %s holding %s at the size the mirror has", repoID, p.version, + p.files[0].File) + } + s.remember(ctx, name, o) + s.checkOne(ctx, m, o) + return o, nil +} + // compare says what the registry's repository has against an origin. func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()} if o.Previous != nil { u.Previous = o.Previous.Revision } - if repo.Revision == o.Revision { + if sameRevision(repo.Revision, o.Revision) { return u } for _, f := range o.Files { @@ -172,6 +294,14 @@ func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { return u } +// sameRevision compares commits either of which may be shortened. +func sameRevision(a, b string) bool { + if a == "" || b == "" { + return a == b + } + return strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + // CheckUpdates asks the registry about every model with an origin. func (s *Server) CheckUpdates(ctx context.Context) { if s.deps.Current == nil { @@ -228,9 +358,16 @@ func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error if err != nil { return UpdateInfo{}, fmt.Errorf("the mirror: %w", err) } + // Asked by hand: one that could not be traced is looked for again. + s.mu.Lock() + if o, ok := s.origins[name]; ok && o == nil { + delete(s.origins, name) + } + s.mu.Unlock() o := s.originOf(ctx, m, list) if o == nil { - return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name) + return UpdateInfo{}, fmt.Errorf("where %s's weights came from is not known: give its repository with "+ + "`llmgw ctl model origin %s OWNER/REPO`", name, name) } return s.checkOne(ctx, m, o), nil } @@ -337,7 +474,7 @@ func (s *Server) runUpdate(j *Job) { s.mu.Unlock() u := compare(m, o, repo) switch { - case repo.Revision == o.Revision: + case sameRevision(repo.Revision, o.Revision): s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision))) return case len(u.Missing) > 0: diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go index 104207f..4c586d8 100644 --- a/internal/control/modelserve/updates_test.go +++ b/internal/control/modelserve/updates_test.go @@ -2,6 +2,7 @@ package modelserve import ( "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -262,3 +263,78 @@ func TestRegistryDownloadURLsAreRead(t *testing.T) { } } } + +// findingRegistry is a registry with history: searchable, and readable at a +// past commit. +type findingRegistry struct { + movingRegistry + found []hf.Model + at map[string]hf.Repo // id@rev +} + +func (r *findingRegistry) Explore(context.Context, hf.Query) ([]hf.Model, error) { return r.found, nil } +func (r *findingRegistry) RepoAt(_ context.Context, id, rev string) (hf.Repo, error) { + if repo, ok := r.at[id+"@"+rev]; ok { + return repo, nil + } + return hf.Repo{}, errors.New("404") +} + +func sibling(names ...string) []struct { + RFilename string `json:"rfilename"` +} { + var out []struct { + RFilename string `json:"rfilename"` + } + for _, n := range names { + out = append(out, struct { + RFilename string `json:"rfilename"` + }{n}) + } + return out +} + +func TestWeightsFetchedBeforeOriginsWereKeptAreTracedToTheirRepository(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + // The mirror kept neither source nor checksum. + mir.held[0].Source, mir.held[0].SHA256 = "", "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, + found: []hf.Model{ + {ID: "copycat/r", Siblings: sibling("q4.gguf")}, // the right name, not the commit + {ID: "other/thing", Siblings: sibling("q8.gguf")}, // not the file + {ID: "o/r", Siblings: sibling("README.md", "q4.gguf")}, + }, + at: map[string]hf.Repo{ + "copycat/r@abc12345": {ID: "copycat/r", Revision: "ffff", Files: []hf.File{{Path: "q4.gguf", Bytes: 1 << 30}}}, + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{ + {Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + }} + s.deps.Registry = fr + s.CheckUpdates(context.Background()) + o := models.models["qwen"].Origin + if o == nil || o.Repo != "o/r" || o.Revision != oldRev || o.Files[0].SHA256 != "aaaa" { + t.Fatalf("traced to %+v", o) + } + if u := s.Updates()["qwen"]; !u.Available { + t.Errorf("%+v", u) + } +} + +func TestAnOriginGivenByHandIsChecked(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + mir.held[0].Source = "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, at: map[string]hf.Repo{ + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + "bad/r@abc12345": {ID: "bad/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", Bytes: 5}}}, + }} + s.deps.Registry = fr + if _, err := s.SetOriginOf(context.Background(), "qwen", "bad/r"); err == nil { + t.Error("a repository whose file is another size was taken") + } + if _, err := s.SetOriginOf(context.Background(), "qwen", "o/r"); err != nil { + t.Fatal(err) + } + if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" { + t.Fatalf("%+v", o) + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index db663f1..2c9edf4 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -68,6 +68,10 @@ type Model struct { // GGUF is what the registry read from the repository's weights, absent // for one that has none. GGUF *GGUFInfo `json:"gguf"` + // Siblings are the repository's files, when asked for with WithFiles. + Siblings []struct { + RFilename string `json:"rfilename"` + } `json:"siblings,omitempty"` } // GGUFInfo is the registry's summary of a repository's GGUF weights. @@ -160,6 +164,9 @@ type Query struct { // "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the // adapters made for one model. Filters []string + // WithFiles lists each repository's files too, to find one holding a + // file by name. + WithFiles bool } // exploreFields is everything a listing shows. Asking for any field with @@ -185,6 +192,9 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) { "direction": {"-1"}, "expand[]": exploreFields, } + if q.WithFiles { + v.Add("expand[]", "siblings") + } if s := strings.TrimSpace(q.Search); s != "" { v.Set("search", s) } @@ -330,7 +340,11 @@ func (r Repo) ShortRevision() string { } // Repo reads a repository's revision and file list. -func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { +func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") } + +// RepoAt reads a repository as it was at a commit, which may be shortened; +// "" is its latest. +func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { id = strings.Trim(strings.TrimSpace(id), "/") if !strings.Contains(id, "/") { return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id) @@ -351,7 +365,11 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // Expanded to what this reads: asking for gguf alone would leave out the // revision and the gate. - if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { + at := "" + if rev != "" { + at = "/revision/" + url.PathEscape(rev) + } + if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { return Repo{}, err } @@ -366,7 +384,7 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // By revision rather than by branch, so the listing and the URLs built from // it describe the same commit even if the branch moves in between. - rev := meta.SHA + rev = meta.SHA if rev == "" { rev = "main" } diff --git a/internal/mirror/add.go b/internal/mirror/add.go index 479e10a..a5d4fb5 100644 --- a/internal/mirror/add.go +++ b/internal/mirror/add.go @@ -101,6 +101,10 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En _ = os.Remove(part) return Added{}, err } + if err := writeSource(final, source); err != nil { + _ = os.Remove(part) + return Added{}, err + } if err := os.Rename(part, final); err != nil { return Added{}, err } @@ -156,6 +160,7 @@ func Remove(root string, e Entry) error { // Its checksum goes with it; left, it would keep the directory from // being empty, and so from going. _ = os.Remove(checksumPath(path)) + _ = os.Remove(sourcePath(path)) versionDir := filepath.Dir(path) if empty, _ := isEmpty(versionDir); empty { _ = os.Remove(versionDir) diff --git a/internal/mirror/manager.go b/internal/mirror/manager.go index a69752c..d85ef36 100644 --- a/internal/mirror/manager.go +++ b/internal/mirror/manager.go @@ -7,6 +7,7 @@ import ( "log/slog" "net/http" "os" + "path/filepath" "sync" "time" ) @@ -289,6 +290,10 @@ func (m *Manager) All() ([]Status, error) { // verifiable — the download finishing was what made it disappear. if k, ok := known[key]; ok { st.SHA256, st.Source = k.SHA256, k.Source + } else { + // Fetched before this process started: what was kept beside it. + path := filepath.Join(m.root, e.Model, e.Version, e.File) + st.SHA256, st.Source = readChecksum(path), readSource(path) } out = append(out, st) seen[key] = struct{}{} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 8b154e8..c2c3b4f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -187,6 +187,31 @@ func writeChecksum(file, sum string) error { return os.Rename(tmp, checksumPath(file)) } +// sourcePath is where the URL a file was fetched from is kept, beside it like +// its checksum: what says, after the mirror restarts, which repository and +// commit a file is, so a newer one there can be noticed. +func sourcePath(file string) string { + return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".source") +} + +func writeSource(file, source string) error { + tmp := sourcePath(file) + partSuffix + if err := os.WriteFile(tmp, []byte(source+"\n"), 0o644); err != nil { //nolint:gosec // a URL, for anyone to read + return err + } + return os.Rename(tmp, sourcePath(file)) +} + +// readSource is where a file was fetched from, or "" for one fetched before +// this was kept, or put in place by hand. +func readSource(file string) string { + b, err := os.ReadFile(sourcePath(file)) //nolint:gosec // beside a file in the mirror + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + // readChecksum is a file's recorded sha256, or "" for a file put in place by // hand, which has none. func readChecksum(file string) string { From 7aefb236d6024605fa4cc1cb205c6b098fac12bd Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:36:27 +0200 Subject: [PATCH 4/5] Publish llmgw_model_update_available, and alert on it Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/monitoring/llmgw-rules.yml | 12 ++++++++++++ internal/cli/server.go | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index 67cacd6..3940ecb 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -156,6 +156,18 @@ groups: llmgw-backup.service` on the control plane says why the last one failed; docs/install.md, "Backing up", says how to set it up. + - alert: ModelUpdateAvailable + expr: llmgw_model_update_available == 1 + for: 1h + labels: { severity: info } + annotations: + summary: "{{ $labels.repo }} has new weights for {{ $labels.model }}" + description: >- + The registry repository the model was served from has published + different files under the same names. `llmgw ctl model updates` + lists what changed; `llmgw ctl model update {{ $labels.model }}` + moves it to them, and `llmgw ctl model rollback` goes back. + - alert: PoolSaturated expr: llmgw_pool_saturation > 1 for: 5m diff --git a/internal/cli/server.go b/internal/cli/server.go index d820e0b..2a79d45 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -184,8 +184,8 @@ func newServerCmd() *cobra.Command { if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil { return err } - // Whether models' repositories have new weights; filled in - // once the mirror is set up, below. + // Whether models' repositories have new weights; given its + // source once the mirror is set up, below. if err := rt.RegisterMetrics(updateMetrics); err != nil { return err } From a16097cfe8faba4a34d53bf2c9bed2a5625aa7eb Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:42:59 +0200 Subject: [PATCH 5/5] Serve vision models with their projector and original weights on vLLM; group the mirror by version model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 19 +++ docs/console.md | 28 ++-- internal/agent/engines/llamacpp.go | 37 ++++- internal/agent/engines/llamacpp_test.go | 33 +++- internal/cli/model_serve.go | 19 ++- internal/cli/tui_console.go | 12 +- internal/cli/tui_hf.go | 103 +++++++++++-- internal/cli/tui_mirror.go | 107 ++++++++++--- internal/control/modelserve/modelserve.go | 145 ++++++++++++++++-- .../control/modelserve/modelserve_test.go | 66 ++++++++ internal/control/modelserve/updates.go | 6 +- internal/hf/hf.go | 15 +- 12 files changed, 509 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5509bd3..e8d1575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,25 @@ is not an update. A model from `server.yaml` is reported but not changed: its source is edited there. +### Models of several files: vision projectors, original weights + +- A vision model's projector (`mmproj`) is fetched beside its GGUF weights + and loaded with them: `llmgw ctl model serve REPO --projector FILE`, or the + choice offered by the console. The model is marked as taking images. Nodes + load it once their agent is this version or later. +- `--format safetensors` serves a repository's original weights, with their + config.json, tokenizer and shard index, on vLLM across `--tp` GPUs of + `--gpu-class` (the fleet's one class when there is one). In the console, + **F** on Serve a model switches to them. +- The mirror keeps where it fetched each file beside it, and reports that + and the checksum after it restarts. The console's Mirror view shows one + row per version of a model, with its files on Enter. +- An update watches every file of a model, the projector and the tokenizer + included; one whose files the mirror cannot trace is found on the + registry by name, commit and size, or given with `llmgw ctl model origin`. +- `llmgw_model_update_available` is published, and `ModelUpdateAvailable` + alerts on it. + ### The manual, in the binary `llmgw docs` reads every document — installation, operation, the diff --git a/docs/console.md b/docs/console.md index 676e27b..978db9c 100644 --- a/docs/console.md +++ b/docs/console.md @@ -111,9 +111,13 @@ which): requests, tokens, errors and spend. team or project is guaranteed and may at most hold, and what it holds now. **A** adds one; on a row, change or remove it. -**Mirror** is what the model mirror holds, how far each download has got, and -which models use each file. **A** downloads a URL into it; on a row, where -nodes fetch it from, or remove it — with a warning when a model still uses it. +**Mirror** is what the model mirror holds, one row for each version of each +model — a file, or the set of files a model of several is: shards, a vision +projector, a config and tokenizer — with how far its download has got and +which models use it. **A** downloads a URL into it; on a row, its files with +their checksums and where each was fetched from, the source a model +definition names for it, or remove it — with a warning when a model still +uses it. **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. @@ -181,12 +185,18 @@ to restart it. Another control plane keeps its own copy. certificate and starts its agent (`sudo llmgw node add`). - **Replicate the store** — moves the store onto two more machines under Patroni (offered only while it is not replicated). -- **Serve a model** — Hugging Face's GGUF models: trending, most downloaded, - most liked or newest (**S** changes which), or **/** to search. Enter on - one lists its files with their sizes; the one chosen is fetched into the - mirror, checked, added to the catalog and started, and a replica is asked - to say hello (`llmgw ctl model serve`). `mirror.registry` points it at - another registry, for a site without the internet. +- **Serve a model** — Hugging Face's models: trending, most downloaded, + most liked or newest (**S** changes which), or **/** to search. **F** + switches between GGUF files, which llama.cpp runs on the CPU or a GPU, and + original safetensors weights, which vLLM runs on GPUs. Enter on a GGUF + repository lists its files with their sizes; a vision model offers its + projector too, which is fetched beside it and loaded with it so the model + reads images. Enter on original weights offers them with their config and + tokenizer, a GPU class and how many GPUs a replica spans. What is chosen is + fetched into the mirror, checked, added to the catalog and started, and a + replica is asked to say hello (`llmgw ctl model serve`). + `mirror.registry` points it at another registry, for a site without the + internet. - **Upgrade the cluster** to a release: this machine, then every node in turn (`sudo llmgw ctl upgrade`). - **Restart all replicas**, one at a time. diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index ad35128..33cadf4 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -93,7 +93,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { bin := l.bin() p := s.Profile - model, err := ggufIn(p.ModelPath) + model, projector, err := ggufIn(p.ModelPath) if err != nil { return nil, err } @@ -120,6 +120,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } + // A vision model's projector, fetched beside it: what turns an image in + // a request into something the language model reads. + if projector != "" && !hasFlag(p.Args, "--mmproj", "-mm") { + args = append(args, "--mmproj", projector) + } if !hasFlag(p.Args, "--parallel", "-np") { n := l.cfg.Parallel if n <= 0 { @@ -203,30 +208,46 @@ func hasFlag(args []string, names ...string) bool { // // A path that is already a file is returned untouched, which is every // single-file model and every hand-staged one. -func ggufIn(path string) (string, error) { +// +// A vision projector in the directory — an mmproj file — is not the model: +// it is returned second, to be loaded beside it. +func ggufIn(path string) (model, projector string, err error) { st, err := os.Stat(path) if err != nil || !st.IsDir() { // Not a directory, or not there at all — the second is the engine's to // report, with the message it would have given anyway. - return path, nil //nolint:nilerr // a missing path is the engine's error to give + return path, "", nil //nolint:nilerr // a missing path is the engine's error to give } entries, err := os.ReadDir(path) if err != nil { - return "", err + return "", "", err } - var names []string + var names, projectors []string for _, e := range entries { - if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { + lower := strings.ToLower(e.Name()) + if e.IsDir() || !strings.HasSuffix(lower, ".gguf") { + continue + } + if strings.Contains(lower, "mmproj") { + projectors = append(projectors, e.Name()) + } else { names = append(names, e.Name()) } } if len(names) == 0 { - return "", fmt.Errorf("%s holds no .gguf file", path) + if len(projectors) > 0 { + return "", "", fmt.Errorf("%s holds a vision projector and no model", path) + } + return "", "", fmt.Errorf("%s holds no .gguf file", path) } // Sorted, so a split model's first part comes first: the parts are named // "-00001-of-00004" and lexical order is shard order. sort.Strings(names) - return filepath.Join(path, names[0]), nil + if len(projectors) > 0 { + sort.Strings(projectors) + projector = filepath.Join(path, projectors[0]) + } + return filepath.Join(path, names[0]), projector, nil } // UseInstalled points the engine at a build the agent installed and checked. diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 07ec11b..0e59cdf 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -204,7 +204,7 @@ func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) { } } - got, err := ggufIn(dir) + got, _, err := ggufIn(dir) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestAPlainFilePathIsUntouched(t *testing.T) { if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { t.Fatal(err) } - if got, _ := ggufIn(p); got != p { + if got, _, _ := ggufIn(p); got != p { t.Errorf("expected the path unchanged, got %s", got) } } @@ -231,7 +231,7 @@ func TestADirectoryWithNoWeightsSaysSo(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil { t.Fatal(err) } - if _, err := ggufIn(dir); err == nil { + if _, _, err := ggufIn(dir); err == nil { t.Error("a directory with no .gguf in it is not a model llama.cpp can load") } } @@ -240,7 +240,7 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) { // The engine's own "no such file" names the path and the flag it came // from, which is more use than anything this could say. p := "/nonexistent/model.gguf" - got, err := ggufIn(p) + got, _, err := ggufIn(p) if err != nil || got != p { t.Errorf("expected it passed through, got %q %v", got, err) } @@ -361,3 +361,28 @@ func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) { t.Error("a one-GPU replica was given a split mode") } } + +func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) { + dir := t.TempDir() + // "mmproj" sorts before the model's name; it must still not be taken for it. + for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}}) + if err != nil { + t.Fatal(err) + } + args := strings.Join(cmds[0].Args, " ") + if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) || + !strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) { + t.Errorf("%s", args) + } + // One the profile names itself is left as it is. + cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir, + Args: []string{"--mmproj", "/elsewhere.gguf"}}}) + if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 { + t.Errorf("two projectors: %v", cmds[0].Args) + } +} diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 8c79501..79e9c27 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" ) @@ -24,15 +25,27 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { "that the nodes' llama.cpp can load it and sizes its memory from its own\n" + "header, adds it to the catalog, waits for a replica, and asks it to say\n" + "hello. Each step is shown as it happens; the first that fails stops it.\n\n" + + "A vision model's projector (--projector) is fetched beside it and loaded with\n" + + "it. --format safetensors serves the original weights instead, with their\n" + + "config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" + " llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" + - " llmgw ctl model serve owner/big-GGUF --on-demand", + " llmgw ctl model serve owner/big-GGUF --on-demand\n" + + " llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" + + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { req.Repo = args[0] if onDemand { req.Replicas = 0 } + // One kind of GPU in the fleet: that one, unless told. + if req.Format == "safetensors" && req.GPUClass == "" { + var sum adminapi.Summary + if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 { + req.GPUClass = sum.GPUs[0].Class + } + } body, _ := json.Marshal(req) var job modelserve.Job if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { @@ -44,6 +57,10 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f := c.Flags() f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)") f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant") + f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it") + f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs") + f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)") + f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans") f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 4d59198..94fbd08 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -292,7 +292,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case consoleMirrorMsg: m.mirror = msg case consoleHFMsg: - if msg.query == m.hf.query && msg.sort == m.hf.sort { + if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors { m.hf.models, m.hf.err = msg.models, msg.err if m.hf.models == nil && msg.err == nil { m.hf.models = []hf.Model{} @@ -447,7 +447,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cQuotas: m.menuD = m.quotaMenu(m.quotas[m.row]) case cMirror: - m.menuD = m.mirrorMenu(m.mirror.res.Models[m.row]) + m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: @@ -464,6 +464,12 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { if m.menu == cServe { m.hfSearchDialog() } + case "f": + if m.menu == cServe { + m.hf.safetensors = !m.hf.safetensors + m.hf.models, m.hf.err, m.row = nil, nil, 0 + return m, m.fetchHF() + } case "s": if m.menu == cServe { m.hf.sort = (m.hf.sort + 1) % len(hfSorts) @@ -824,7 +830,7 @@ func (m consoleModel) rowCount() int { case cQuotas: return len(m.quotas) case cMirror: - return len(m.mirror.res.Models) + return len(m.mirrorGroups()) case cServe: return len(m.hf.models) case cAccounts: diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 4b106bb..18eab3f 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -11,6 +11,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" ) @@ -24,15 +25,19 @@ var hfSorts = []struct { }{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}} type hfState struct { - query string - sort int - models []hf.Model - err error + query string + sort int + // safetensors lists original weights, for vLLM on GPUs, rather than + // GGUF files for llama.cpp. + safetensors bool + models []hf.Model + err error } type consoleHFMsg struct { query string sort int + st bool models []hf.Model err error } @@ -56,18 +61,21 @@ func (m consoleModel) registry() *hf.Client { } func (m consoleModel) fetchHF() tea.Cmd { - c, q, sort := m.registry(), m.hf.query, m.hf.sort + c, q, sort, st := m.registry(), m.hf.query, m.hf.sort, m.hf.safetensors return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50} + hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: !st, Limit: 50} + if st { + hq.Filters = []string{"safetensors"} + } // Browsing: models that chat. A search finds the rest too, an // embedding model included. if q == "" { hq.Task = "text-generation" } got, err := c.Explore(ctx, hq) - return consoleHFMsg{query: q, sort: sort, models: got, err: err} + return consoleHFMsg{query: q, sort: sort, st: st, models: got, err: err} } } @@ -85,6 +93,17 @@ func (m *consoleModel) fetchRepo(id string) tea.Cmd { // repoMenu offers a repository's GGUF files, smallest first as listed. func (m consoleModel) repoMenu(r hf.Repo) *menuDialog { var items []menuItem + if m.hf.safetensors { + title := r.ID + set, err := modelserve.SafetensorsSet(r) + if err != nil { + return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}} + } + return &menuDialog{title: title, items: []menuItem{{ + fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)), + func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil }, + }}} + } for _, set := range hf.Group(r.Files) { if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") { continue @@ -124,13 +143,19 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { note += ", licence " + r.License } note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes." + fields := []formField{ + textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("replicas", "1", "0 runs none until it is first asked for"), + textF("context length", "4096", "what one request may use"), + textF("description", "", "optional: shown to clients in /v1/models"), + } + // A vision model: its projector, loaded beside it, is what reads images. + if proj := modelserve.Projectors(r); len(proj) > 0 { + fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0])) + fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model" + } m.openForm(&formDialog{title: "Serve " + r.ID, note: note, - fields: []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), - textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "4096", "what one request may use"), - textF("description", "", "optional: shown to clients in /v1/models"), - }, + fields: fields, submit: func(m *consoleModel, v []string) (tea.Cmd, error) { n, err := strconv.Atoi(v[1]) if err != nil || n < 0 { @@ -147,6 +172,50 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { if v[3] != "" { args = append(args, "--description="+v[3]) } + if len(v) > 4 && !strings.HasPrefix(v[4], "none") { + args = append(args, "--projector="+v[4]) + } + return m.runAfter(args...), nil + }}) +} + +func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) { + server := "--server=" + m.client.base + var classes []string + for _, g := range m.data.sum.GPUs { + classes = append(classes, g.Class) + } + if len(classes) == 0 { + m.dialog, m.dialogTitle = "error", "No GPUs" + m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" + + "F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key") + return + } + m.openForm(&formDialog{title: "Serve " + r.ID, + note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+ + "Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)), + fields: []formField{ + textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("replicas", "1", "0 runs none until it is first asked for"), + textF("context length", "8192", "what one request may use; vLLM's --max-model-len"), + textF("description", "", "optional: shown to clients in /v1/models"), + choiceF("GPU class", classes, classes[0]), + textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} { + if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) { + return nil, fmt.Errorf("%s is a whole number", what) + } + } + args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4], + "--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} + if v[1] == "0" { + args = append(args, "--on-demand") + } + if v[3] != "" { + args = append(args, "--description="+v[3]) + } return m.runAfter(args...), nil }}) } @@ -164,7 +233,7 @@ func (m *consoleModel) hfSearchDialog() { func (m consoleModel) hfPanel(pw int) (string, string, string) { keys := "→ or Enter open · ↑↓ choose · Q leave" if m.inPanel { - keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave" + keys = "↑↓ row · Enter choose · / search · S order · F GGUF or original weights · ← back · Q leave" } what := hfSorts[m.hf.sort].label if m.hf.query != "" { @@ -173,7 +242,11 @@ func (m consoleModel) hfPanel(pw int) (string, string, string) { if m.hf.query == "" { what = "text generation, " + what } - head := "\n GGUF models on Hugging Face: " + what + "\n\n" + kind := "GGUF models, for llama.cpp" + if m.hf.safetensors { + kind = "Original weights, for vLLM on GPUs" + } + head := "\n " + kind + " on Hugging Face: " + what + "\n\n" switch { case m.hf.err != nil: return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) + diff --git a/internal/cli/tui_mirror.go b/internal/cli/tui_mirror.go index bd2295a..68d0f69 100644 --- a/internal/cli/tui_mirror.go +++ b/internal/cli/tui_mirror.go @@ -62,26 +62,75 @@ func (m *consoleModel) mirrorCall(title string, do func(context.Context, *mirror } } -// usedBy names the catalog's models whose source is this file. -func (m consoleModel) usedBy(base string, s mirror.Status) []string { - u := mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(base) - dir := strings.TrimSuffix(mirror.Entry{Model: s.Model, Version: s.Version}.URL(base), "/") + "/" +// mirrorGroup is one version of one model on the mirror: a file, or the set +// of files a model of several is fetched as. +type mirrorGroup struct { + model, version string + files []mirror.Status +} + +func (g mirrorGroup) bytes() (done, total int64) { + for _, f := range g.files { + done += f.BytesDone + total += max(f.BytesTotal, f.BytesDone) + } + return done, total +} + +// source is what a model's definition names for it: the file, or the +// directory of a set. +func (g mirrorGroup) source(base string) string { + if len(g.files) == 1 { + return mirror.Entry{Model: g.model, Version: g.version, File: g.files[0].File}.URL(base) + } + return strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/" +} + +func (m consoleModel) mirrorGroups() []mirrorGroup { + var out []mirrorGroup + at := map[string]int{} + for _, st := range m.mirror.res.Models { + k := st.Model + "/" + st.Version + i, ok := at[k] + if !ok { + i = len(out) + at[k] = i + out = append(out, mirrorGroup{model: st.Model, version: st.Version}) + } + out[i].files = append(out[i].files, st) + } + return out +} + +// usedBy names the catalog's models that run from a version's files. +func (m consoleModel) usedBy(base string, g mirrorGroup) []string { + dir := strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/" var names []string for _, md := range m.data.models { - if md.Source == u || strings.HasPrefix(md.Source, dir) { + if strings.HasPrefix(md.Source, dir) { names = append(names, md.Name) } } return names } -func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog { - what := s.Model + "/" + s.Version - used := m.usedBy(m.mirror.res.Base, s) - items := []menuItem{{"Where nodes fetch it", func(m *consoleModel) tea.Cmd { +func (m consoleModel) mirrorMenu(g mirrorGroup) *menuDialog { + what := g.model + "/" + g.version + used := m.usedBy(m.mirror.res.Base, g) + items := []menuItem{{"Its files, and where nodes fetch them", func(m *consoleModel) tea.Cmd { + var b strings.Builder + for _, f := range g.files { + sum := "" + if f.SHA256 != "" { + sum = sDim.Render(" sha256 " + f.SHA256[:min(12, len(f.SHA256))]) + } + fmt.Fprintf(&b, "%-50s %9s%s\n", truncate(f.File, 50), bytesShort(max(f.BytesTotal, f.BytesDone)), sum) + if f.Source != "" { + b.WriteString(sDim.Render(" from "+truncate(f.Source, 90)) + "\n") + } + } m.dialog, m.dialogTitle = "error", what - m.dialogText = mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(m.mirror.res.Base) + - "\n\n" + sDim.Render("the source to give a model definition · any key") + m.dialogText = b.String() + "\nA model's source: " + g.source(m.mirror.res.Base) + "\n\n" + sDim.Render("any key") return nil }}} items = append(items, menuItem{"Remove from the mirror…", func(m *consoleModel) tea.Cmd { @@ -93,7 +142,7 @@ func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog { m.confirmD = &confirmDialog{title: "Remove " + what, text: text, yes: func(m *consoleModel) tea.Cmd { return m.mirrorCall("Remove "+what, func(ctx context.Context, c *mirror.Client) (string, error) { - removed, err := c.Remove(ctx, s.Model, s.Version) + removed, err := c.Remove(ctx, g.model, g.version) return fmt.Sprintf("Removed %d file(s).", len(removed)), err }) }} @@ -139,24 +188,32 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) { } head := "\n " + sDim.Render("Nodes fetch from "+m.mirror.res.Base) + "\n\n" var rows [][]string - for _, s := range m.mirror.res.Models { - state := string(s.State) - switch s.State { - case mirror.Fetching: - if p, ok := s.Percent(); ok { - state = sWarn.Render(fmt.Sprintf("fetching %d%%", p)) - } else { - state = sWarn.Render("fetching " + bytesShort(s.BytesDone)) + for _, g := range m.mirrorGroups() { + done, total := g.bytes() + state := "present" + for _, f := range g.files { + switch f.State { + case mirror.Fetching: + state = sWarn.Render("fetching") + if total > 0 { + state = sWarn.Render(fmt.Sprintf("fetching %d%%", done*100/total)) + } + case mirror.Failed: + state = sBad.Render("failed") + } + if f.State != mirror.Present { + break } - case mirror.Failed: - state = sBad.Render("failed") } - size := bytesShort(s.BytesDone) - used := strings.Join(m.usedBy(m.mirror.res.Base, s), ", ") + what := g.files[0].File + if len(g.files) > 1 { + what = fmt.Sprintf("%d files", len(g.files)) + } + used := strings.Join(m.usedBy(m.mirror.res.Base, g), ", ") if used == "" { used = sDim.Render("unused") } - rows = append(rows, []string{s.Model, s.Version, s.File, state, size, used}) + rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used}) } if len(rows) == 0 { return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index d68491d..8cda83b 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -87,6 +87,15 @@ type Request struct { Quant string `json:"quant,omitempty"` // File picks it by name instead. File string `json:"file,omitempty"` + // Projector is a vision model's mmproj file in the repository, fetched + // beside the weights and loaded with them; empty serves text alone. + Projector string `json:"projector,omitempty"` + // Format is "gguf", run by llama.cpp, or "safetensors": the original + // weights with their config and tokenizer, run by vLLM on GPUs of + // GPUClass, across TP of them. + Format string `json:"format,omitempty"` + GPUClass string `json:"gpu_class,omitempty"` + TP int `json:"tp,omitempty"` // Name is the catalog name; the repository suggests one otherwise. Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` @@ -179,6 +188,20 @@ func (s *Server) Start(req Request) (Job, error) { if req.Replicas < 0 { return Job{}, errors.New("replicas cannot be negative") } + switch req.Format { + case "", "gguf": + req.Format = "gguf" + case "safetensors": + if req.GPUClass == "" { + return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it") + } + if req.Projector != "" { + return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files") + } + req.TP = max(req.TP, 1) + default: + return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) + } if req.Quant == "" && req.File == "" { req.Quant = "Q4_K_M" } @@ -251,11 +274,28 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo)) return } - set, err := pick(repo, req) + pickSet := pick + if req.Format == "safetensors" { + pickSet = pickSafetensors + } + set, err := pickSet(repo, req) if err != nil { s.fail(j, "registry", err) return } + if req.Projector != "" { + i := slices.IndexFunc(repo.Files, func(f hf.File) bool { + return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) && + strings.Contains(strings.ToLower(f.Path), "mmproj") + }) + if i < 0 { + s.fail(j, "registry", fmt.Errorf("%s has no vision projector %q; it has: %s", repo.ID, req.Projector, + strings.Join(Projectors(repo), ", "))) + return + } + set.Files = append(set.Files, repo.Files[i]) + set.Bytes += repo.Files[i].Bytes + } name := req.Name if name == "" { name = hf.ModelName(repo.ID) @@ -274,10 +314,20 @@ func (s *Server) run(j *Job) { } // 3. Whether the nodes' llama.cpp loads it, and how much memory it needs. + if req.Format == "safetensors" { + // vLLM reads the model's shape from its config.json, which came + // with the weights; the GPUs it needs are the class and width asked. + s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass)) + s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ + Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100, + Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)}, + }) + return + } s.step(j, "check", Running, "reading its header from the mirror") var header gguf.Header - if !strings.HasSuffix(source, "/") { - v := gguf.Check(ctx, source, s.deps.LlamaVersion) + if u := headerURL(source, set); u != "" { + v := gguf.Check(ctx, u, s.deps.LlamaVersion) if !v.Servable { s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason)) return @@ -302,16 +352,24 @@ func (s *Server) run(j *Job) { } } + s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ + Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, + Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, + }) +} + +// addAndStart puts the model in the catalog with its profile, waits for a +// replica and asks it to answer. +func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) { + req := j.Request // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, - Profiles: []catalog.Profile{{ - Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, - }}, - Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, - Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, + Profiles: []catalog.Profile{prof}, + Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, + Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, + Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { @@ -593,3 +651,72 @@ func orUnknown(arch string) string { } return arch } + +// Projectors are a repository's vision projectors: the mmproj files a vision +// model loads beside its weights. +func Projectors(repo hf.Repo) []string { + var out []string + for _, f := range repo.Files { + if strings.HasSuffix(strings.ToLower(f.Path), ".gguf") && strings.Contains(strings.ToLower(f.Path), "mmproj") { + out = append(out, f.Path) + } + } + return out +} + +// headerURL is the file whose header says what the model is: the source +// itself, or in a directory the first part of the weights, not a projector. +func headerURL(source string, set hf.FileSet) string { + if !strings.HasSuffix(source, "/") { + return source + } + var names []string + for _, f := range set.Files { + name := mirror.LastPathElement(f.Path) + lower := strings.ToLower(name) + if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") { + names = append(names, name) + } + } + if len(names) == 0 { + return "" + } + slices.Sort(names) + return source + names[0] +} + +// SafetensorsSet is what serving a repository's original weights fetches. +func SafetensorsSet(repo hf.Repo) (hf.FileSet, error) { return pickSafetensors(repo, Request{}) } + +// pickSafetensors chooses a repository's original weights and the files an +// engine reads beside them: config.json, the tokenizer, the shard index. +func pickSafetensors(repo hf.Repo, _ Request) (hf.FileSet, error) { + var weights []hf.File + for _, f := range repo.Files { + if !strings.Contains(f.Path, "/") && strings.HasSuffix(f.Path, ".safetensors") { + weights = append(weights, f) + } + } + sets := hf.Group(weights) + if len(sets) == 0 { + return hf.FileSet{}, fmt.Errorf("%s has no safetensors weights at its top", repo.ID) + } + // Sharded "model-00001-of-00004" or a single "model.safetensors"; a + // repository with a second, consolidated copy keeps it under another name. + set := sets[0] + for _, st := range sets { + if strings.HasPrefix(st.Name, "model") { + set = st + break + } + } + companions := hf.Companions(repo.Files) + if !slices.ContainsFunc(companions, func(f hf.File) bool { return f.Path == "config.json" }) { + return hf.FileSet{}, fmt.Errorf("%s has no config.json, which vLLM reads the model's shape from", repo.ID) + } + for _, f := range companions { + set.Files = append(set.Files, f) + set.Bytes += f.Bytes + } + return set, nil +} diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go index 7c4fdfe..2760628 100644 --- a/internal/control/modelserve/modelserve_test.go +++ b/internal/control/modelserve/modelserve_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -223,3 +224,68 @@ func TestARepositoryThatIsNotOneIsRefused(t *testing.T) { } } } + +func TestAVisionModelIsServedWithItsProjectorBesideIt(t *testing.T) { + files := []hf.File{ + {Path: "gemma-3-4b-it-Q4_K_M.gguf", Bytes: 2 << 30, SHA256: "aa"}, + {Path: "mmproj-gemma-3-4b-it-f16.gguf", Bytes: 800 << 20, SHA256: "bb"}, + } + s, m, c, _ := setup(t, files, map[string]string{"gemma-3-4b-it-Q4_K_M.gguf": "aa", "mmproj-gemma-3-4b-it-f16.gguf": "bb"}) + job, err := s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", + Projector: "mmproj-gemma-3-4b-it-f16.gguf", Replicas: 1}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + if len(m.got) != 2 { + t.Fatalf("fetched %+v", m.got) + } + got := c.added[0] + if !strings.HasSuffix(got.Source, "/") || !got.Capabilities.Vision || len(got.Origin.Files) != 2 { + t.Errorf("%s vision=%v origin=%+v", got.Source, got.Capabilities.Vision, got.Origin) + } + // A projector the repository does not have is named with those it has. + job, _ = s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", Projector: "mmproj-x.gguf", Name: "g2"}) + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "mmproj-gemma-3-4b-it-f16.gguf") { + t.Errorf("%+v", j) + } +} + +func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) { + files := []hf.File{ + {Path: "model-00001-of-00002.safetensors", Bytes: 4 << 30, SHA256: "a1"}, + {Path: "model-00002-of-00002.safetensors", Bytes: 3 << 30, SHA256: "a2"}, + {Path: "model.safetensors.index.json", Bytes: 30000}, + {Path: "config.json", Bytes: 700}, + {Path: "tokenizer.json", Bytes: 7 << 20}, + {Path: "tokenizer_config.json", Bytes: 7000}, + {Path: "README.md", Bytes: 9000}, + {Path: "original/consolidated.safetensors", Bytes: 7 << 30}, + } + s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"}) + if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil { + t.Error("safetensors with no GPU class was taken") + } + job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + var got []string + for _, r := range m.got { + got = append(got, r.File) + } + if len(got) != 6 || slices.Contains(got, "README.md") || slices.Contains(got, "consolidated.safetensors") { + t.Errorf("fetched %v", got) + } + mod := c.added[0] + p := mod.Profiles[0] + if !strings.HasSuffix(mod.Source, "/") || p.Engine != "vllm" || p.GPUClass != "l40s-48g" || p.TP != 1 || + !slices.Contains(p.Args, "--max-model-len") { + t.Errorf("%s %+v", mod.Source, p) + } +} diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index 62bebb6..c34615a 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -509,9 +509,9 @@ func (s *Server) runUpdate(j *Job) { // 3. Whether the nodes load it, and whether it still fits what is // reserved for it. s.step(j, "check", Running, "reading its header from the mirror") - detail := "a set of shards; not read" - if !strings.HasSuffix(source, "/") { - v := gguf.Check(ctx, source, s.deps.LlamaVersion) + detail := "no header to read" + if u := headerURL(source, set); u != "" { + v := gguf.Check(ctx, u, s.deps.LlamaVersion) if !v.Servable { s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) return diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 2c9edf4..f67d548 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -68,6 +68,10 @@ type Model struct { // GGUF is what the registry read from the repository's weights, absent // for one that has none. GGUF *GGUFInfo `json:"gguf"` + // Safetensors is the registry's summary of original weights. + Safetensors *struct { + Total int64 `json:"total"` + } `json:"safetensors,omitempty"` // Siblings are the repository's files, when asked for with WithFiles. Siblings []struct { RFilename string `json:"rfilename"` @@ -84,10 +88,13 @@ type GGUFInfo struct { // Params is the parameter count, 0 when the registry does not know it. func (m Model) Params() int64 { - if m.GGUF == nil { - return 0 + switch { + case m.GGUF != nil: + return m.GGUF.Total + case m.Safetensors != nil: + return m.Safetensors.Total } - return m.GGUF.Total + return 0 } // Architecture is the GGUF architecture, empty when unknown. @@ -172,7 +179,7 @@ type Query struct { // exploreFields is everything a listing shows. Asking for any field with // expand returns only the fields asked for, so all of them are named. var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified", - "createdAt", "gated", "private", "trendingScore", "gguf"} + "createdAt", "gated", "private", "trendingScore", "gguf", "safetensors"} // Explore lists repositories: the trending ones, the newest, or a search. //