From 44425193bd53469949b25f5fb07ba72dbcb6fdfb Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:12:20 +0200 Subject: [PATCH] Track the registry's new weights for served models, and move a model to them or back A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/reference.md | 93 ++++ internal/cli/model.go | 3 +- internal/cli/model_serve.go | 57 +- internal/cli/model_update.go | 119 ++++ internal/cli/server.go | 17 +- internal/cli/tui_actions.go | 56 ++ internal/cli/tui_console.go | 19 +- internal/config/config.go | 6 + internal/control/adminapi/api.go | 94 ++++ internal/control/catalog/catalog.go | 45 ++ internal/control/modelserve/modelserve.go | 34 +- internal/control/modelserve/updates.go | 569 ++++++++++++++++++++ internal/control/modelserve/updates_test.go | 264 +++++++++ internal/control/modelsvc/modelsvc.go | 32 ++ internal/docgen/reference_gen.go | 18 + 15 files changed, 1393 insertions(+), 33 deletions(-) create mode 100644 internal/cli/model_update.go create mode 100644 internal/control/modelserve/updates.go create mode 100644 internal/control/modelserve/updates_test.go diff --git a/docs/reference.md b/docs/reference.md index b8606a7..ab6c09b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -76,6 +76,7 @@ rather than repeated. | `tls` | `tlsconf.Config` | [Mirror](#mirror) | | `registry` | `string` | [Mirror](#mirror) | | `timeout` | `time.Duration` | [Mirror](#mirror) | +| `update_check` | `time.Duration` | [Mirror](#mirror) | | `name` | `string` | [Model](#model) | | `aliases` | `[]string` | [Model](#model) | | `description` | `string` | [Model](#model) | @@ -92,6 +93,7 @@ rather than repeated. | `context_length` | `int` | [Model](#model) | | `max_output` | `int` | [Model](#model) | | `capabilities` | `Capabilities` | [Model](#model) | +| `origin` | `*Origin` | [Model](#model) | | `prometheus` | `string` | [Monitoring](#monitoring) | | `window` | `time.Duration` | [Monitoring](#monitoring) | | `timeout` | `time.Duration` | [Monitoring](#monitoring) | @@ -99,6 +101,13 @@ rather than repeated. | `ib_root` | `string` | [Options](#options) | | `prefer_nvml` | `bool` | [Options](#options) | | `skip_gpus` | `bool` | [Options](#options) | +| `repo` | `string` | [Origin](#origin) | +| `revision` | `string` | [Origin](#origin) | +| `files` | `[]OriginFile` | [Origin](#origin) | +| `previous` | `*PreviousOrigin` | [Origin](#origin) | +| `path` | `string` | [OriginFile](#originfile) | +| `sha256` | `string` | [OriginFile](#originfile) | +| `bytes` | `int64` | [OriginFile](#originfile) | | `disabled` | `bool` | [Personal](#personal) | | `rpm` | `int` | [Personal](#personal) | | `tpm` | `int` | [Personal](#personal) | @@ -111,6 +120,9 @@ rather than repeated. | `match_ratio` | `float64` | [Prefix](#prefix) | | `balance_abs` | `float64` | [Prefix](#prefix) | | `balance_rel` | `float64` | [Prefix](#prefix) | +| `source` | `string` | [PreviousOrigin](#previousorigin) | +| `revision` | `string` | [PreviousOrigin](#previousorigin) | +| `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) | | `name` | `string` | [Profile](#profile) | | `engine` | `string` | [Profile](#profile) | | `gpu_class` | `string` | [Profile](#profile) | @@ -668,6 +680,15 @@ Timeout bounds a management call. It does not bound a download: the mirror answers a fetch as soon as it has started one, and the bytes keep arriving long after this deadline has passed. +### `update_check` + +`time.Duration` + +UpdateCheck is how often the registry is asked whether the files of +the models served from it have changed there: a repository can publish +new weights under the same name. Only told, never applied — moving a +model to them is `llmgw ctl model update`. 0 never asks. + ## Model ### `name` @@ -791,6 +812,19 @@ Capabilities is what this model can do beyond plain completion. Takes a [Capabilities](#capabilities) block. +### `origin` + +`*Origin` + +Origin is the registry repository the weights came from, at which +commit, and each file's checksum there. Written when a model is served +from the registry; for one that was not, worked out from the mirror's +record of where it fetched the file. It is what lets the control plane +say that the repository has published different weights since, and +move the model to them. + +Takes a [Origin](#origin) block. + ## Monitoring ### `prometheus` @@ -845,6 +879,51 @@ is used, is the more exact source of the peers. SkipGPUs omits GPU discovery. A node without nvidia-smi is already found to have none, so this is for one whose driver is broken. +## Origin + +### `repo` + +`string` + +Repo is the repository, owner/name. + +### `revision` + +`string` + +Revision is the commit the files were fetched at. + +### `files` + +`[]OriginFile` + +Files are the weights fetched from it, with what the registry +published as each one's sha256. A new commit that leaves them alone — +a README edited — is not an update. + +### `previous` + +`*PreviousOrigin` + +Previous is what the last update replaced, kept so it can be gone back +to while its files are still on the mirror. + +Takes a [PreviousOrigin](#previousorigin) block. + +## OriginFile + +### `path` + +`string` + +### `sha256` + +`string` + +### `bytes` + +`int64` + ## Personal ### `disabled` @@ -910,6 +989,20 @@ before affinity is abandoned. `float64` +## PreviousOrigin + +### `source` + +`string` + +### `revision` + +`string` + +### `files` + +`[]OriginFile` + ## Profile ### `name` diff --git a/internal/cli/model.go b/internal/cli/model.go index f60b854..94e4f48 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -35,7 +35,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), - newModelServeCmd(client)) + newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), + newModelRollbackCmd(client)) return c } diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 174913b..8c79501 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -38,32 +38,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { return err } - out := cmd.OutOrStdout() - fmt.Fprintf(out, "job %s\n", job.ID) - shown := map[string]string{} - for { - for _, s := range job.Steps { - line := s.State + " " + s.Detail - if shown[s.Name] == line { - continue - } - shown[s.Name] = line - mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok", - modelserve.Failed: "!!"}[s.State] - fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail) - } - switch job.State { - case modelserve.Done: - fmt.Fprintf(out, "\n%s is served.\n", job.Model) - return nil - case modelserve.Failed: - return errors.New(job.Error) - } - time.Sleep(2 * time.Second) - if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil { - return err - } - } + return followJob(cmd, client, job, "is served") }, } f := c.Flags() @@ -79,3 +54,33 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use") return c } + +// followJob shows a job's steps as they happen, until it ends. +func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, doneSays string) error { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "job %s\n", job.ID) + shown := map[string]string{} + for { + for _, s := range job.Steps { + line := s.State + " " + s.Detail + if shown[s.Name] == line { + continue + } + shown[s.Name] = line + mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok", + modelserve.Failed: "!!", modelserve.Skipped: "--"}[s.State] + fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail) + } + switch job.State { + case modelserve.Done: + fmt.Fprintf(out, "\n%s %s.\n", job.Model, doneSays) + return nil + case modelserve.Failed: + return errors.New(job.Error) + } + time.Sleep(2 * time.Second) + if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil { + return err + } + } +} diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go new file mode 100644 index 0000000..4111663 --- /dev/null +++ b/internal/cli/model_update.go @@ -0,0 +1,119 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" +) + +func newModelUpdatesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + var check bool + c := &cobra.Command{ + Use: "updates [MODEL]", + Short: "Show which models' repositories have published new weights", + Long: "A model served from the registry remembers the repository and commit its\n" + + "files came from, and the control plane asks the registry every\n" + + "mirror.update_check (six hours by default) whether those files have\n" + + "changed there. A new commit that leaves them alone is not an update.\n\n" + + "--check asks now rather than showing the last answer.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var list []modelserve.UpdateInfo + if check || len(args) == 1 { + req := map[string]string{} + if len(args) == 1 { + req["model"] = args[0] + } + body, _ := json.Marshal(req) + if err := client().post(cmd.Context(), "/api/v1/updates/check", body, &list); err != nil { + return err + } + } else if err := client().get(cmd.Context(), "/api/v1/updates", &list); err != nil { + return err + } + if *asJSON { + return emitJSON(list) + } + if len(list) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no model has an origin to compare yet; one is checked a few minutes after the control plane starts") + return nil + } + w := tabTo(cmd.OutOrStdout()) + fmt.Fprintln(w, "MODEL\tREPOSITORY\tRUNS\tLATEST\tUPDATE\tCHECKED") + for _, u := range list { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", u.Model, u.Repo, catalog.ShortRevision(u.Current), + catalog.ShortRevision(u.Latest), updateCell(u), ago(u.CheckedAt, time.Now())) + } + if err := w.Flush(); err != nil { + return err + } + for _, u := range list { + if u.Available { + fmt.Fprintf(cmd.OutOrStdout(), "\nTo move one to its new weights: llmgw ctl model update MODEL\n") + break + } + } + return nil + }, + } + c.Flags().BoolVar(&check, "check", false, "ask the registry now") + return c +} + +// updateCell says what an update would be, or why there is none. +func updateCell(u modelserve.UpdateInfo) string { + switch { + case u.Err != "": + return "could not ask: " + truncate(u.Err, 60) + case u.Available: + return fmt.Sprintf("yes: %d file(s), %s", len(u.Changed), bytesShort(u.Bytes)) + case len(u.Missing) > 0: + return "the repository no longer has " + strings.Join(u.Missing, ", ") + case u.Latest != "" && u.Latest != u.Current: + return "no: new commit, same weights" + } + return "-" +} + +func newModelUpdateCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "update MODEL", + Short: "Move a model to the weights its repository publishes now", + Long: "The new files are fetched into the mirror beside the ones in use and\n" + + "checked against the registry's checksums and the nodes' llama.cpp; then\n" + + "the catalog names them and the replicas are replaced one at a time, each\n" + + "new one up before an old one stops, and one is asked to say hello.\n\n" + + "The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" + + "back to them. A model from server.yaml is changed there instead.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var job modelserve.Job + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil { + return err + } + return followJob(cmd, client, job, "runs the new weights") + }, + } +} + +func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "rollback MODEL", + Short: "Move a model back to the weights its last update replaced", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var out map[string]string + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/rollback", []byte("{}"), &out); err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), out["result"]) + return nil + }, + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 4688a26..d820e0b 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -128,6 +128,7 @@ func newServerCmd() *cobra.Command { var agentHandler http.Handler var agentPath string var admin *control.Controller + updateMetrics := &modelserve.UpdateCollector{} // The store's cluster, when it is one; read by the metrics and // by `ctl status`. var storeHA *pgha.Collector @@ -183,6 +184,11 @@ func newServerCmd() *cobra.Command { if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil { return err } + // Whether models' repositories have new weights; filled in + // once the mirror is set up, below. + if err := rt.RegisterMetrics(updateMetrics); err != nil { + return err + } // A store of several hosts is a replicated cluster: what state // it is in is published beside everything else, read from its // Patroni. One host is a single PostgreSQL, which has none. @@ -321,10 +327,17 @@ func newServerCmd() *cobra.Command { // One command from a registry repository to a replica that // answers: `llmgw ctl model serve`. if adminAPI != nil && catalogSvc != nil { - adminAPI.SetServe(modelserve.New(modelserve.Deps{ + serve := modelserve.New(modelserve.Deps{ Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin, LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log, - })) + // Updates: the catalog in force, the service that + // changes it, and what replaces replicas. + Current: func() []catalog.Model { return admin.Catalog().Models() }, + Models: catalogSvc, Restart: admin.Reconciler().Restart, + }) + adminAPI.SetServe(serve) + updateMetrics.Set(serve) + go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck) } log.Info("model mirror configured", "api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled()) diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 47cd17f..05a0784 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -418,6 +419,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog { }) }}, } + items = append(items, m.updateItems(md)...) if md.Parked { items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd { return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil) @@ -674,3 +676,57 @@ func (m *consoleModel) doEntry() tea.Cmd { } return nil } + +// updateItems are what can be done about a model's repository publishing new +// weights: move to them, ask now, or go back. +func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { + if md.Origin == nil { + return nil + } + server := "--server=" + m.client.base + var items []menuItem + if u := md.Update; u != nil && u.Available { + items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)), + func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Update " + md.Name, + text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+ + "They are fetched into the mirror beside the ones in use and checked, then the\n"+ + "replicas are replaced one at a time. The old files stay, to go back to.", + u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")), + yes: func(m *consoleModel) tea.Cmd { return m.runAfter("ctl", server, "model", "update", md.Name) }} + return nil + }}) + } + items = append(items, menuItem{"Ask the registry for new weights now", func(m *consoleModel) tea.Cmd { + return m.call("New weights for "+md.Name, http.MethodPost, "/api/v1/updates/check", map[string]string{"model": md.Name}, + func(raw json.RawMessage) string { + var us []modelserve.UpdateInfo + if json.Unmarshal(raw, &us) != nil || len(us) == 0 { + return string(raw) + } + u := us[0] + if u.Available { + return fmt.Sprintf("%s has new weights at %s: %s, %s.", u.Repo, catalog.ShortRevision(u.Latest), + strings.Join(u.Changed, ", "), bytesShort(u.Bytes)) + } + return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u)) + }) + }}) + if prev := md.Origin.Previous; prev != "" { + items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Go back on " + md.Name, + text: "It runs the weights its last update replaced, " + catalog.ShortRevision(prev) + + ", if they are still on the mirror;\nits replicas are replaced one at a time.", + yes: func(m *consoleModel) tea.Cmd { + return m.call("Go back on "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/rollback", + map[string]string{}, func(raw json.RawMessage) string { + var out map[string]string + _ = json.Unmarshal(raw, &out) + return out["result"] + "." + }) + }} + return nil + }}) + } + return items +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index c48022a..ce7e888 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -19,6 +19,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -959,13 +960,17 @@ func (m consoleModel) panel() (title, body, keys string) { if len(md.Warnings) > 0 { warn = sWarn.Render("!") } + upd := "" + if md.Update != nil && md.Update.Available { + upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest)) + } rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class, - strings.Join(md.Profiles, ","), warn}) + strings.Join(md.Profiles, ","), upd, warn}) } if m.inPanel { keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave" } - return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", ""}, rows, m.row, pw, m.inPanel), keys + return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys case cReplicas: var rows [][]string for _, r := range m.data.replicas { @@ -1174,6 +1179,16 @@ func (m consoleModel) modelDetail(md adminapi.ModelView) string { kv("profiles", strings.Join(md.Profiles, ", ")) kv("aliases", strings.Join(md.Aliases, ", ")) kv("source", md.Source) + if o := md.Origin; o != nil { + from := o.Repo + " at " + catalog.ShortRevision(o.Revision) + if o.Previous != "" { + from += ", updated from " + catalog.ShortRevision(o.Previous) + } + kv("from", from) + } + if u := md.Update; u != nil { + kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now()))) + } if md.MemoryGB > 0 { kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB)) } diff --git a/internal/config/config.go b/internal/config/config.go index a01980b..bf09a06 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -177,6 +177,11 @@ type Mirror struct { // mirror answers a fetch as soon as it has started one, and the bytes keep // arriving long after this deadline has passed. Timeout time.Duration `koanf:"timeout"` + // UpdateCheck is how often the registry is asked whether the files of + // the models served from it have changed there: a repository can publish + // new weights under the same name. Only told, never applied — moving a + // model to them is `llmgw ctl model update`. 0 never asks. + UpdateCheck time.Duration `koanf:"update_check"` } // Landing configures the public page at "/". @@ -481,6 +486,7 @@ func Default() Server { Listen: ":8080", AdminListen: "127.0.0.1:9090", AgentListen: ":7443", + Mirror: Mirror{UpdateCheck: 6 * time.Hour}, Control: Control{ Interval: 5 * time.Second, BaseBackoff: 10 * time.Second, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 3a18611..a32d851 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -121,6 +121,10 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/restart", a.restart) mux.HandleFunc("POST /api/v1/models/serve", a.serveModel) mux.HandleFunc("GET /api/v1/jobs/{id}", a.job) + mux.HandleFunc("GET /api/v1/updates", a.listUpdates) + mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) + mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) + mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -336,6 +340,19 @@ type ModelView struct { Adapters []catalog.Adapter `json:"adapters,omitempty"` Preempted int `json:"preempted,omitempty"` HeldFor string `json:"held_for,omitempty"` + // Origin is the registry repository its weights came from, and at which + // commit; Update what that repository had when it was last asked. + Origin *OriginView `json:"origin,omitempty"` + Update *modelserve.UpdateInfo `json:"update,omitempty"` +} + +// OriginView is where a model's weights were published. +type OriginView struct { + Repo string `json:"repo"` + Revision string `json:"revision"` + // Previous is the commit the last update replaced, which it can go back + // to. + Previous string `json:"previous,omitempty"` } // ReplicaView is a replica as `ctl replicas` shows it. @@ -586,6 +603,10 @@ func (a *API) Models() []ModelView { } measured[rep.Model] = max(measured[rep.Model], b) } + var updates map[string]modelserve.UpdateInfo + if a.serve != nil { + updates = a.serve.Updates() + } out := make([]ModelView, 0) for _, m := range a.ctl.Catalog().Models() { h := health[m.Name] @@ -605,6 +626,15 @@ func (a *API) Models() []ModelView { if len(m.Profiles) > 0 { v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores } + if o := m.Origin; o != nil { + v.Origin = &OriginView{Repo: o.Repo, Revision: o.Revision} + if o.Previous != nil { + v.Origin.Previous = o.Previous.Revision + } + } + if u, ok := updates[m.Name]; ok { + v.Update = &u + } v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30) v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB) out = append(out, v) @@ -1426,6 +1456,70 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, job) } +func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) { + out := make([]modelserve.UpdateInfo, 0) + if a.serve != nil { + for _, u := range a.serve.Updates() { + out = append(out, u) + } + } + slices.SortFunc(out, func(x, y modelserve.UpdateInfo) int { return strings.Compare(x.Model, y.Model) }) + writeJSON(w, http.StatusOK, out) +} + +// CheckRequest asks the registry about one model now, or every one. +type CheckRequest struct { + Model string `json:"model,omitempty"` +} + +func (a *API) checkUpdates(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + var req CheckRequest + _ = json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req) + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute) + defer cancel() + if req.Model != "" { + u, err := a.serve.CheckModel(ctx, req.Model) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, []modelserve.UpdateInfo{u}) + return + } + a.serve.CheckUpdates(ctx) + a.listUpdates(w, r) +} + +func (a *API) updateModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusAccepted, job) +} + +func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + msg, err := a.serve.Rollback(r.Context(), r.PathValue("name"), actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"result": msg}) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 70c599e..710e5c5 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -194,6 +194,51 @@ type Model struct { // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` + + // Origin is the registry repository the weights came from, at which + // commit, and each file's checksum there. Written when a model is served + // from the registry; for one that was not, worked out from the mirror's + // record of where it fetched the file. It is what lets the control plane + // say that the repository has published different weights since, and + // move the model to them. + Origin *Origin `json:"origin,omitempty" koanf:"origin"` +} + +// Origin is where a model's weights were published. +type Origin struct { + // Repo is the repository, owner/name. + Repo string `json:"repo" koanf:"repo"` + // Revision is the commit the files were fetched at. + Revision string `json:"revision" koanf:"revision"` + // Files are the weights fetched from it, with what the registry + // published as each one's sha256. A new commit that leaves them alone — + // a README edited — is not an update. + Files []OriginFile `json:"files,omitempty" koanf:"files"` + // Previous is what the last update replaced, kept so it can be gone back + // to while its files are still on the mirror. + Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"` +} + +// OriginFile is one weights file of an origin. +type OriginFile struct { + Path string `json:"path" koanf:"path"` + SHA256 string `json:"sha256,omitempty" koanf:"sha256"` + Bytes int64 `json:"bytes,omitempty" koanf:"bytes"` +} + +// PreviousOrigin is a model's source before an update, and where it came from. +type PreviousOrigin struct { + Source string `json:"source" koanf:"source"` + Revision string `json:"revision" koanf:"revision"` + Files []OriginFile `json:"files,omitempty" koanf:"files"` +} + +// ShortRevision is a commit as a person writes it. +func ShortRevision(rev string) string { + if len(rev) > 8 { + return rev[:8] + } + return rev } // Capabilities describes what a client may ask a model for. diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 6d22a2d..3ef4807 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -71,6 +71,12 @@ type Deps struct { // ReadyTimeout bounds the wait for a replica: the node fetching the // weights from the mirror and the engine loading them. ReadyTimeout time.Duration + + // For updates: the catalog in force, the service that changes it, and + // what replaces replicas. Without them updates are not offered. + Current func() []catalog.Model + Models Models + Restart func(ids ...string) error } // Request is what to serve. @@ -116,7 +122,10 @@ type Step struct { // Job is one model being served, and where it stands. type Job struct { - ID string `json:"id"` + ID string `json:"id"` + // Kind is "update" for a model moved to new weights, empty for one + // served. + Kind string `json:"kind,omitempty"` Request Request `json:"request"` State string `json:"state"` Error string `json:"error,omitempty"` @@ -135,6 +144,8 @@ type Server struct { mu sync.Mutex jobs map[string]*Job seq int + // updates is what the last checks of the registry found, by model. + updates map[string]UpdateInfo } // New builds a server. @@ -151,7 +162,7 @@ func New(d Deps) *Server { if d.Log == nil { d.Log = slog.Default() } - return &Server{deps: d, jobs: map[string]*Job{}} + return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}} } var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) @@ -297,6 +308,7 @@ func (s *Server) run(j *Job) { Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, }}, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, + Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { @@ -388,8 +400,26 @@ func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSe version := repo.ShortRevision() s.step(j, "mirror", Running, "asking the mirror for "+set.Name) var base string + // Files already there from this commit, checksums matching, are not + // fetched again: a model moved back and then forward again. + have := map[string]bool{} + if got, err := s.deps.Mirror.List(ctx); err == nil { + for _, st := range got.Models { + if st.Model == name && st.Version == version && st.State == mirror.Present { + for _, f := range set.Files { + if mirror.LastPathElement(f.Path) == st.File && (f.SHA256 == "" || strings.EqualFold(f.SHA256, st.SHA256)) { + have[st.File] = true + base = mirror.Entry{Model: name, Version: version, File: st.File}.URL(got.Base) + } + } + } + } + } for _, f := range set.Files { file := mirror.LastPathElement(f.Path) + if have[file] { + continue + } resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{ Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file, }) diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go new file mode 100644 index 0000000..cc446e8 --- /dev/null +++ b/internal/control/modelserve/updates.go @@ -0,0 +1,569 @@ +package modelserve + +import ( + "context" + "errors" + "fmt" + "net/url" + "slices" + "strings" + "sync/atomic" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "github.com/prometheus/client_golang/prometheus" +) + +// Updates: a repository publishes new weights under the same name — a fixed +// chat template, a better quantisation — and nothing told anyone. A model +// served from the registry now remembers where it came from; this asks the +// registry, every so often, whether the files it runs have changed there, +// and moves a model to them on request: fetched into the mirror beside the +// old ones, checked, swapped in, its replicas replaced one at a time, and +// the old files kept so it can go back. + +// Models is the catalog as an update changes it. +type Models interface { + Declared(name string) bool + SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) + SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error +} + +// UpdateInfo is what the registry has for a model against what it runs. +type UpdateInfo struct { + Model string `json:"model"` + Repo string `json:"repo,omitempty"` + // Current is the commit the model runs, Latest the repository's now. + Current string `json:"current,omitempty"` + Latest string `json:"latest,omitempty"` + // Changed are the model's files whose weights differ at Latest, and + // Missing those Latest no longer has at all. + Changed []string `json:"changed,omitempty"` + Missing []string `json:"missing,omitempty"` + // Bytes is what an update downloads. + Bytes int64 `json:"bytes,omitempty"` + // Available is true when the model can be moved to Latest. + Available bool `json:"available"` + // Previous is the commit an update replaced, which it can go back to. + Previous string `json:"previous,omitempty"` + CheckedAt time.Time `json:"checked_at"` + Err string `json:"error,omitempty"` +} + +// place is where a model's weights sit on the mirror. +type place struct { + model, version string + files []mirror.Status +} + +// placeOf finds a source on the mirror: a file, or a directory of shards. +func placeOf(source string, list mirror.ModelsResponse) (place, bool) { + base := strings.TrimSuffix(list.Base, "/") + "/" + if list.Base == "" || !strings.HasPrefix(source, base) { + return place{}, false + } + parts := strings.SplitN(strings.TrimPrefix(source, base), "/", 3) + if len(parts) < 2 { + return place{}, false + } + p := place{model: parts[0], version: parts[1]} + file := "" + if len(parts) == 3 { + file = parts[2] + } + for _, st := range list.Models { + if st.Model == p.model && st.Version == p.version && (file == "" || st.File == file) { + p.files = append(p.files, st) + } + } + return p, len(p.files) > 0 +} + +// parseResolve reads a registry download URL: base/owner/name/resolve/rev/path. +func parseResolve(u string) (repo, rev, path string, ok bool) { + pu, err := url.Parse(u) + if err != nil { + return "", "", "", false + } + i := strings.Index(pu.Path, "/resolve/") + if i < 0 { + return "", "", "", false + } + repo = strings.Trim(pu.Path[:i], "/") + if strings.Count(repo, "/") != 1 { + return "", "", "", false + } + rev, path, ok = strings.Cut(pu.Path[i+len("/resolve/"):], "/") + return repo, rev, path, ok && rev != "" && path != "" +} + +// inferOrigin works a model's origin out from what the mirror recorded of +// where it fetched each file, for a model served before origins were kept. +func inferOrigin(p place) (*catalog.Origin, bool) { + var o catalog.Origin + for _, st := range p.files { + repo, rev, path, ok := parseResolve(st.Source) + if !ok || (o.Repo != "" && (repo != o.Repo || rev != o.Revision)) { + return nil, false + } + o.Repo, o.Revision = repo, rev + o.Files = append(o.Files, catalog.OriginFile{Path: path, SHA256: st.SHA256, Bytes: st.BytesDone}) + } + return &o, o.Repo != "" +} + +func originFiles(set hf.FileSet) []catalog.OriginFile { + out := make([]catalog.OriginFile, 0, len(set.Files)) + for _, f := range set.Files { + out = append(out, catalog.OriginFile{Path: f.Path, SHA256: f.SHA256, Bytes: f.Bytes}) + } + return out +} + +// originOf is a model's origin: recorded, or worked out from the mirror and +// then recorded. +func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin { + if m.Origin != nil && m.Origin.Repo != "" { + return m.Origin + } + p, ok := placeOf(m.Source, list) + if !ok { + return nil + } + o, ok := inferOrigin(p) + if !ok { + return nil + } + if s.deps.Models != nil { + if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil { + s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err) + } + } + return o +} + +// compare says what the registry's repository has against an origin. +func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { + u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()} + if o.Previous != nil { + u.Previous = o.Previous.Revision + } + if repo.Revision == o.Revision { + return u + } + for _, f := range o.Files { + i := slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path }) + if i < 0 { + u.Missing = append(u.Missing, f.Path) + continue + } + g := repo.Files[i] + if (f.SHA256 != "" && g.SHA256 != "" && !strings.EqualFold(f.SHA256, g.SHA256)) || + (f.SHA256 == "" && f.Bytes != 0 && f.Bytes != g.Bytes) { + u.Changed = append(u.Changed, f.Path) + u.Bytes += g.Bytes + } + } + u.Available = len(u.Changed) > 0 && len(u.Missing) == 0 + return u +} + +// CheckUpdates asks the registry about every model with an origin. +func (s *Server) CheckUpdates(ctx context.Context) { + if s.deps.Current == nil { + return + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + s.deps.Log.Warn("checking for model updates: the mirror", "err", err) + return + } + seen := map[string]bool{} + for _, m := range s.deps.Current() { + o := s.originOf(ctx, m, list) + if o == nil { + continue + } + seen[m.Name] = true + s.checkOne(ctx, m, o) + } + s.mu.Lock() + for name := range s.updates { + if !seen[name] { + delete(s.updates, name) + } + } + s.mu.Unlock() +} + +func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origin) UpdateInfo { + repo, err := s.deps.Registry.Repo(ctx, o.Repo) + u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, CheckedAt: time.Now()} + if err != nil { + u.Err = err.Error() + } else { + u = compare(m, o, repo) + if u.Available { + s.deps.Log.Info("a model's repository has new weights", "model", m.Name, "repo", o.Repo, + "current", catalog.ShortRevision(o.Revision), "latest", catalog.ShortRevision(repo.Revision)) + } + } + s.mu.Lock() + s.updates[m.Name] = u + s.mu.Unlock() + return u +} + +// CheckModel asks the registry about one model now. +func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error) { + m, err := s.current(name) + if err != nil { + return UpdateInfo{}, err + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return UpdateInfo{}, fmt.Errorf("the mirror: %w", err) + } + o := s.originOf(ctx, m, list) + if o == nil { + return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name) + } + return s.checkOne(ctx, m, o), nil +} + +// Updates is what the last checks found, by model. +func (s *Server) Updates() map[string]UpdateInfo { + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]UpdateInfo, len(s.updates)) + for k, v := range s.updates { + out[k] = v + } + return out +} + +// WatchUpdates checks every so often until ctx ends; zero never checks. +func (s *Server) WatchUpdates(ctx context.Context, every time.Duration) { + if every <= 0 { + return + } + // A little after starting, when the catalog is in force and the mirror + // answering, rather than in the middle of starting. + wait := 2 * time.Minute + for { + select { + case <-ctx.Done(): + return + case <-time.After(wait): + } + s.CheckUpdates(ctx) + wait = every + } +} + +func (s *Server) current(name string) (catalog.Model, error) { + if s.deps.Current == nil { + return catalog.Model{}, errors.New("this control plane cannot read its catalog here") + } + for _, m := range s.deps.Current() { + if m.Name == name { + return m, nil + } + } + return catalog.Model{}, fmt.Errorf("no model called %s", name) +} + +// StartUpdate moves a model to its repository's latest weights, as a job. +func (s *Server) StartUpdate(name, by string) (Job, error) { + if s.deps.Models == nil || s.deps.Restart == nil { + return Job{}, errors.New("updating needs the store, which this control plane lacks") + } + if _, err := s.current(name); err != nil { + return Job{}, err + } + if s.deps.Models.Declared(name) { + return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+ + "or they would revert on the next restart", name) + } + s.mu.Lock() + for _, j := range s.jobs { + if j.Kind == "update" && j.Model == name && j.State == Running { + s.mu.Unlock() + return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID) + } + } + s.seq++ + j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name, + Request: Request{Name: name, By: by}, State: Running, Started: time.Now()} + s.jobs[j.ID] = j + s.mu.Unlock() + go s.runUpdate(j) + return s.snapshot(j), nil +} + +func (s *Server) runUpdate(j *Job) { + ctx := context.Background() + name := j.Model + + // 1. The registry: which files, at which commit. + s.step(j, "registry", Running, "reading where "+name+" came from") + m, err := s.current(name) + if err != nil { + s.fail(j, "registry", err) + return + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + s.fail(j, "registry", fmt.Errorf("the mirror: %w", err)) + return + } + o := s.originOf(ctx, m, list) + p, onMirror := placeOf(m.Source, list) + if o == nil || !onMirror { + s.fail(j, "registry", fmt.Errorf("%s did not come from the registry through the mirror", name)) + return + } + repo, err := s.deps.Registry.Repo(ctx, o.Repo) + if err != nil { + s.fail(j, "registry", err) + return + } + s.mu.Lock() + j.Request.Repo = repo.ID + s.mu.Unlock() + u := compare(m, o, repo) + switch { + case repo.Revision == o.Revision: + s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision))) + return + case len(u.Missing) > 0: + s.fail(j, "registry", fmt.Errorf("%s at %s no longer has %s; choose one of its files and serve it under a new name", + repo.ID, repo.ShortRevision(), strings.Join(u.Missing, ", "))) + return + case len(u.Changed) == 0: + s.fail(j, "registry", fmt.Errorf("%s has a newer commit, %s, but the files %s runs are the same in it", + repo.ID, repo.ShortRevision(), name)) + return + case repo.Gated.Is(): + s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", repo.ID)) + return + } + set := hf.FileSet{} + for _, f := range o.Files { + g := repo.Files[slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })] + set.Files = append(set.Files, g) + set.Bytes += g.Bytes + } + set.Name, set.Shards = set.Files[0].Path, len(set.Files) + s.step(j, "registry", Done, fmt.Sprintf("%s: %s → %s, %d of %d file(s) changed, %.1f GB", + repo.ID, catalog.ShortRevision(o.Revision), repo.ShortRevision(), len(u.Changed), len(o.Files), gb(set.Bytes))) + + // 2. The mirror, beside the files in use. + source, err := s.mirror(ctx, j, repo, set, p.model) + if err != nil { + s.fail(j, "mirror", err) + return + } + + // 3. Whether the nodes load it, and whether it still fits what is + // reserved for it. + s.step(j, "check", Running, "reading its header from the mirror") + detail := "a set of shards; not read" + if !strings.HasSuffix(source, "/") { + v := gguf.Check(ctx, source, s.deps.LlamaVersion) + if !v.Servable { + s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) + return + } + detail = orUnknown(v.Header.Architecture) + if len(m.Profiles) > 0 { + ctxPool := 16384 + for i, a := range m.Profiles[0].Args { + if a == "--ctx-size" && i+1 < len(m.Profiles[0].Args) { + fmt.Sscan(m.Profiles[0].Args[i+1], &ctxPool) + } + } + need := int((v.Header.MemoryBytes(set.Bytes, ctxPool) + 1<<30 - 1) >> 30) + if have := m.Profiles[0].MinMemoryGB; need > have && have > 0 { + detail += fmt.Sprintf("; needs about %d GB and %d is reserved: raise it if replicas fail", need, have) + } + } + } + s.step(j, "check", Done, detail) + + // 4. The catalog: the new source, and the old one kept to go back to. + s.step(j, "catalog", Running, "moving "+name+" to "+repo.ShortRevision()) + next := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set), + Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: o.Revision, Files: o.Files}} + if _, err := s.deps.Models.SetSource(ctx, name, source, next, j.Request.By); err != nil { + s.fail(j, "catalog", err) + return + } + s.step(j, "catalog", Done, "the catalog names "+source) + + // 5. The replicas, replaced one at a time. + if err := s.replace(ctx, j, name); err != nil { + s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back to %s", err, name, + catalog.ShortRevision(o.Revision))) + return + } + s.mu.Lock() + s.updates[name] = UpdateInfo{Model: name, Repo: repo.ID, Current: repo.Revision, Latest: repo.Revision, + Previous: o.Revision, CheckedAt: time.Now()} + s.mu.Unlock() + s.finish(j) +} + +// replace restarts a model's replicas onto the catalog in force and waits +// for the new ones, then asks one to answer. +func (s *Server) replace(ctx context.Context, j *Job, name string) error { + var old []string + running := 0 + for _, r := range s.deps.Fleet.Replicas() { + if r.Model != name { + continue + } + running++ + if r.State == reconciler.Ready { + old = append(old, r.ID) + } + } + if running == 0 { + s.step(j, "replicas", Skipped, "none running; the next to start loads the new weights") + return nil + } + if len(old) == 0 { + s.step(j, "replicas", Skipped, "none ready; those starting load the new weights when they next start") + return nil + } + if err := s.deps.Restart(old...); err != nil { + return err + } + s.step(j, "replicas", Running, fmt.Sprintf("replacing %d, one at a time", len(old))) + deadline := time.Now().Add(s.deps.ReadyTimeout) + for { + left, ready := 0, []reconciler.Replica{} + for _, r := range s.deps.Fleet.Replicas() { + if r.Model != name { + continue + } + if slices.Contains(old, r.ID) { + left++ + } else if r.State == reconciler.Ready { + ready = append(ready, r) + } + } + for _, h := range s.deps.Fleet.ModelHealth() { + if h.Model == name && h.Parked { + return fmt.Errorf("the new weights failed to start %d times: %s", h.Failures, h.Reason) + } + } + if left == 0 && len(ready) > 0 { + s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old))) + s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello") + reply, tps, err := s.smoke(ctx, ready[0], name) + if err != nil { + s.step(j, "smoke test", Failed, err.Error()) + return err + } + s.mu.Lock() + j.Reply, j.TokensPerSecond = reply, tps + s.mu.Unlock() + s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply)) + return nil + } + s.step(j, "replicas", Running, fmt.Sprintf("%d of %d replaced, %d new ready", len(old)-left, len(old), len(ready))) + if time.Now().After(deadline) { + return fmt.Errorf("the replicas were not replaced after %s", s.deps.ReadyTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.deps.Poll): + } + } +} + +// Rollback moves a model back to what its last update replaced, if those +// files are still on the mirror, and restarts its replicas onto them. The +// update is kept as the one to go forward to again. +func (s *Server) Rollback(ctx context.Context, name, by string) (string, error) { + if s.deps.Models == nil || s.deps.Restart == nil { + return "", errors.New("going back needs the store, which this control plane lacks") + } + m, err := s.current(name) + if err != nil { + return "", err + } + if m.Origin == nil || m.Origin.Previous == nil { + return "", fmt.Errorf("%s has not been updated, so there is nothing to go back to", name) + } + prev := m.Origin.Previous + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return "", fmt.Errorf("the mirror: %w", err) + } + if _, ok := placeOf(prev.Source, list); !ok { + return "", fmt.Errorf("the files of %s are no longer on the mirror", catalog.ShortRevision(prev.Revision)) + } + back := &catalog.Origin{Repo: m.Origin.Repo, Revision: prev.Revision, Files: prev.Files, + Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: m.Origin.Revision, Files: m.Origin.Files}} + if _, err := s.deps.Models.SetSource(ctx, name, prev.Source, back, by); err != nil { + return "", err + } + var ids []string + for _, r := range s.deps.Fleet.Replicas() { + if r.Model == name && r.State == reconciler.Ready { + ids = append(ids, r.ID) + } + } + msg := fmt.Sprintf("%s runs %s again", name, catalog.ShortRevision(prev.Revision)) + if len(ids) > 0 { + if err := s.deps.Restart(ids...); err != nil { + return "", err + } + msg += fmt.Sprintf("; %d replica(s) are being replaced", len(ids)) + } + s.mu.Lock() + if u, ok := s.updates[name]; ok { + u.Current, u.Previous = prev.Revision, m.Origin.Revision + u.Available = u.Latest != "" && u.Latest != prev.Revision + s.updates[name] = u + } + s.mu.Unlock() + return msg, nil +} + +// UpdateCollector publishes llmgw_model_update_available, 1 for each model +// whose repository has new weights. Registered when the metrics are, before +// there is a server to read: Set gives it one. +type UpdateCollector struct{ s atomic.Pointer[Server] } + +var updateDesc = prometheus.NewDesc("llmgw_model_update_available", + "1 when the registry repository a model was served from has published different weights for its files.", + []string{"model", "repo"}, nil) + +// Set gives the collector the server whose checks it reports. +func (c *UpdateCollector) Set(s *Server) { c.s.Store(s) } + +// Describe implements prometheus.Collector. +func (c *UpdateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- updateDesc } + +// Collect implements prometheus.Collector. +func (c *UpdateCollector) Collect(ch chan<- prometheus.Metric) { + s := c.s.Load() + if s == nil { + return + } + for _, u := range s.Updates() { + v := 0.0 + if u.Available { + v = 1 + } + ch <- prometheus.MustNewConstMetric(updateDesc, prometheus.GaugeValue, v, u.Model, u.Repo) + } +} diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go new file mode 100644 index 0000000..104207f --- /dev/null +++ b/internal/control/modelserve/updates_test.go @@ -0,0 +1,264 @@ +package modelserve + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +const ( + oldRev = "abc123456789aaaa" + newRev = "def987654321bbbb" +) + +type movingRegistry struct { + mu sync.Mutex + repo hf.Repo +} + +func (r *movingRegistry) Repo(context.Context, string) (hf.Repo, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.repo, nil +} + +// updMirror is a mirror that records where it fetched each file from. +type updMirror struct { + mu sync.Mutex + base string + held []mirror.Status + sums map[string]string +} + +func (m *updMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.held = append(m.held, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, Source: r.Source, + State: mirror.Present, BytesDone: 1 << 30, SHA256: m.sums[r.Version+"/"+r.File]}) + return mirror.FetchResponse{URL: mirror.Entry{Model: r.Model, Version: r.Version, File: r.File}.URL(m.base)}, nil +} + +func (m *updMirror) List(context.Context) (mirror.ModelsResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + return mirror.ModelsResponse{Base: m.base, Models: append([]mirror.Status{}, m.held...)}, nil +} + +type updModels struct { + mu sync.Mutex + models map[string]catalog.Model + declared map[string]bool +} + +func (c *updModels) Declared(n string) bool { return c.declared[n] } +func (c *updModels) SetSource(_ context.Context, n, src string, o *catalog.Origin, _ string) (catalog.Model, error) { + c.mu.Lock() + defer c.mu.Unlock() + m := c.models[n] + m.Source, m.Origin = src, o + c.models[n] = m + return m, nil +} +func (c *updModels) SetOrigin(_ context.Context, n string, o *catalog.Origin) error { + c.mu.Lock() + defer c.mu.Unlock() + m := c.models[n] + m.Origin = o + c.models[n] = m + return nil +} +func (c *updModels) all() []catalog.Model { + c.mu.Lock() + defer c.mu.Unlock() + var out []catalog.Model + for _, m := range c.models { + out = append(out, m) + } + return out +} + +// restartFleet replaces a restarted replica with a new one, ready. +type restartFleet struct { + mu sync.Mutex + reps []reconciler.Replica + engine string + seq int +} + +func (f *restartFleet) Replicas() []reconciler.Replica { + f.mu.Lock() + defer f.mu.Unlock() + return append([]reconciler.Replica{}, f.reps...) +} +func (f *restartFleet) ModelHealth() []reconciler.ModelHealth { return nil } +func (f *restartFleet) EngineKey(string) string { return "" } +func (f *restartFleet) Restart(ids ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + for _, id := range ids { + for i, r := range f.reps { + if r.ID == id { + f.seq++ + f.reps[i] = reconciler.Replica{ID: r.Model + "-new" + string(rune('0'+f.seq)), Model: r.Model, + State: reconciler.Ready, Endpoint: f.engine} + } + } + } + return nil +} + +func updSetup(t *testing.T) (*Server, *updMirror, *updModels, *movingRegistry, *restartFleet) { + t.Helper() + header := ggufHeader() + files := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(header) })) + t.Cleanup(files.Close) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hi"}}],"usage":{"completion_tokens":1}}`)) + })) + t.Cleanup(engine.Close) + + mir := &updMirror{base: files.URL, sums: map[string]string{ + "abc12345/q4.gguf": "aaaa", "def98765/q4.gguf": "bbbb"}} + // What was fetched before origins were kept: the mirror alone says where from. + _, _ = mir.Fetch(context.Background(), mirror.FetchRequest{Model: "qwen", Version: "abc12345", File: "q4.gguf", + Source: "https://huggingface.co/o/r/resolve/" + oldRev + "/q4.gguf"}) + models := &updModels{models: map[string]catalog.Model{"qwen": {Name: "qwen", + Source: files.URL + "/qwen/abc12345/q4.gguf", + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}}}, declared: map[string]bool{}} + reg := &movingRegistry{repo: hf.Repo{ID: "o/r", Revision: newRev, Files: []hf.File{ + {Path: "q4.gguf", SHA256: "bbbb", Bytes: 2 << 30}, {Path: "README.md", Bytes: 10}}}} + fleet := &restartFleet{engine: engine.URL, reps: []reconciler.Replica{ + {ID: "qwen-1", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}, + {ID: "qwen-2", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}}} + s := New(Deps{Registry: reg, Mirror: mir, Catalog: &fakeCatalog{}, Fleet: fleet, + Current: models.all, Models: models, Restart: fleet.Restart, + Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second}) + return s, mir, models, reg, fleet +} + +func TestNewWeightsInTheRepositoryAreFoundForAModelServedBeforeOriginsWereKept(t *testing.T) { + s, _, models, _, _ := updSetup(t) + s.CheckUpdates(context.Background()) + u := s.Updates()["qwen"] + if !u.Available || u.Repo != "o/r" || u.Current != oldRev || u.Latest != newRev || + len(u.Changed) != 1 || u.Changed[0] != "q4.gguf" || u.Bytes != 2<<30 { + t.Fatalf("%+v", u) + } + if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" || o.Files[0].SHA256 != "aaaa" { + t.Errorf("the origin worked out was not recorded: %+v", o) + } +} + +func TestANewCommitThatLeavesTheWeightsAloneIsNotAnUpdate(t *testing.T) { + s, mir, _, _, _ := updSetup(t) + mir.held[0].SHA256 = "bbbb" + u, err := s.CheckModel(context.Background(), "qwen") + if err != nil || u.Available || u.Latest != newRev || updateSays(u) != "same" { + t.Fatalf("%+v %v", u, err) + } +} + +func updateSays(u UpdateInfo) string { + if !u.Available && u.Latest != u.Current && len(u.Missing) == 0 { + return "same" + } + return "" +} + +func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) { + s, mir, models, _, fleet := updSetup(t) + oldSource := models.models["qwen"].Source + job, err := s.StartUpdate("qwen", "tester") + if err != nil { + t.Fatal(err) + } + j := wait(t, s, job.ID) + if j.State != Done { + t.Fatalf("%s: %+v", j.Error, j.Steps) + } + m := models.models["qwen"] + if !strings.HasSuffix(m.Source, "/qwen/def98765/q4.gguf") || m.Origin.Revision != newRev || + m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource { + t.Fatalf("catalog: %s %+v", m.Source, m.Origin) + } + if len(mir.held) != 2 { + t.Errorf("the old files did not stay on the mirror beside the new: %+v", mir.held) + } + for _, r := range fleet.Replicas() { + if !strings.Contains(r.ID, "new") { + t.Errorf("%s was not replaced", r.ID) + } + } + if u := s.Updates()["qwen"]; u.Available || u.Current != newRev { + t.Errorf("still offered: %+v", u) + } + + // And back. + if _, err := s.Rollback(context.Background(), "qwen", "tester"); err != nil { + t.Fatal(err) + } + m = models.models["qwen"] + if m.Source != oldSource || m.Origin.Revision != oldRev || m.Origin.Previous.Revision != newRev { + t.Fatalf("rollback: %s %+v", m.Source, m.Origin) + } + if u := s.Updates()["qwen"]; !u.Available { + t.Errorf("the update it went back from is not offered again: %+v", u) + } + + // And forward again, from files already on the mirror. + job, err = s.StartUpdate("qwen", "tester") + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done || models.models["qwen"].Origin.Revision != newRev { + t.Fatalf("%s: %+v", j.Error, j.Steps) + } + // Then there is nothing newer: refused, saying why. + job, _ = s.StartUpdate("qwen", "tester") + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") { + t.Fatalf("%+v", j) + } +} + +func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) { + s, _, models, _, _ := updSetup(t) + models.declared["qwen"] = true + if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") { + t.Fatalf("%v", err) + } +} + +func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) { + s, _, _, reg, _ := updSetup(t) + reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}} + job, _ := s.StartUpdate("qwen", "t") + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") { + t.Fatalf("%+v", j) + } +} + +func TestRegistryDownloadURLsAreRead(t *testing.T) { + for u, want := range map[string][3]string{ + "https://huggingface.co/bartowski/Qwen-GGUF/resolve/abc/sub/f.gguf": {"bartowski/Qwen-GGUF", "abc", "sub/f.gguf"}, + "https://hf.internal/mirror/x/resolve/main/f.gguf": {"mirror/x", "main", "f.gguf"}, + } { + repo, rev, path, ok := parseResolve(u) + if !ok || [3]string{repo, rev, path} != want { + t.Errorf("%s: %s %s %s %v", u, repo, rev, path, ok) + } + } + for _, u := range []string{"https://example.com/f.gguf", "https://huggingface.co/a/b/c/resolve/x/f", ":"} { + if _, _, _, ok := parseResolve(u); ok { + t.Errorf("%s was read", u) + } + } +} diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 303653f..b45be61 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -288,6 +288,38 @@ func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catal return final, err } +// SetSource moves a model added at runtime to other weights, recording where +// they came from. Running replicas keep what they loaded until they are +// replaced; the caller restarts them. +func (s *Service) SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its source", func(m *catalog.Model) error { + m.Source, m.Origin = source, origin + return nil + }) + if err == nil { + rev := "" + if origin != nil { + rev = origin.Revision + } + s.log.Info("model source set", "model", name, "source", source, "revision", rev, "by", by) + } + return final, err +} + +// SetOrigin records where a model's weights came from without changing them: +// for a model added before origins were kept. A model from configuration is +// left alone, its origin worked out again when it is needed. +func (s *Service) SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error { + if s.Declared(name) { + return nil + } + _, err := s.update(ctx, name, "llmgw", "record its origin", func(m *catalog.Model) error { + m.Origin = origin + return nil + }) + return err +} + // update changes one model added at runtime and keeps the change. // // Built before anything is written, like Add: catalog.New is what applies the diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 347f83b..408eb59 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -86,6 +86,7 @@ var Reference = []Block{ {Key: "tls", Type: "tlsconf.Config", Doc: "TLS authenticates to the mirror's management API, with the same CA the\nagents use. Empty leaves the connection in plain HTTP, which is only\nacceptable on a network where nothing untrusted can reach that port —\nthe mirror says as much in its own log when it starts that way.", Nested: "tlsconf.Config"}, {Key: "registry", Type: "string", Doc: "Registry is the model registry the mirror page browses, empty for the\npublic Hugging Face. Set it to an internal mirror of the registry where\nthe control plane cannot reach the internet — or leave the whole mirror\nblock out, which turns the explorer off rather than giving it a search\nbox that can only time out.", Nested: ""}, {Key: "timeout", Type: "time.Duration", Doc: "Timeout bounds a management call. It does not bound a download: the\nmirror answers a fetch as soon as it has started one, and the bytes keep\narriving long after this deadline has passed.", Nested: ""}, + {Key: "update_check", Type: "time.Duration", Doc: "UpdateCheck is how often the registry is asked whether the files of\nthe models served from it have changed there: a repository can publish\nnew weights under the same name. Only told, never applied — moving a\nmodel to them is `llmgw ctl model update`. 0 never asks.", Nested: ""}, }}, {Package: "catalog", Name: "Model", Title: "Model", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, @@ -104,6 +105,7 @@ var Reference = []Block{ {Key: "context_length", Type: "int", Doc: "ContextLength is how many tokens one request may use, prompt and\ncompletion together, and MaxOutput bounds the completion alone. Zero\nmeans unknown, which is what every model was until now.\n\nThe gateway knew neither, so an over-long prompt was not refused: it was\nadmitted, given a replica, and failed at the engine, and a client sizing\nits prompts from /v1/models had nothing to read. Worse, the number is not\nthe one an operator writes in the engine's arguments — llama.cpp divides\n--ctx-size across its parallel slots, so a pool of 8192 across four slots\nis 2048 a request, and nothing said so.", Nested: ""}, {Key: "max_output", Type: "int", Doc: "", Nested: ""}, {Key: "capabilities", Type: "Capabilities", Doc: "Capabilities is what this model can do beyond plain completion.", Nested: "catalog.Capabilities"}, + {Key: "origin", Type: "*Origin", Doc: "Origin is the registry repository the weights came from, at which\ncommit, and each file's checksum there. Written when a model is served\nfrom the registry; for one that was not, worked out from the mirror's\nrecord of where it fetched the file. It is what lets the control plane\nsay that the repository has published different weights since, and\nmove the model to them.", Nested: "catalog.Origin"}, }}, {Package: "config", Name: "Monitoring", Title: "Monitoring", Doc: "", Fields: []Field{ {Key: "prometheus", Type: "string", Doc: "Prometheus is the server's address, for example\nhttp://monitoring.internal:9090. Empty leaves the charts off the\ndashboard rather than showing an error where one should be.", Nested: ""}, @@ -116,6 +118,17 @@ var Reference = []Block{ {Key: "prefer_nvml", Type: "bool", Doc: "PreferNVML uses the NVML backend when this binary has one. NVLink\ntopology is read from nvidia-smi topo -m on every build; NVML, where it\nis used, is the more exact source of the peers.", Nested: ""}, {Key: "skip_gpus", Type: "bool", Doc: "SkipGPUs omits GPU discovery. A node without nvidia-smi is already\nfound to have none, so this is for one whose driver is broken.", Nested: ""}, }}, + {Package: "catalog", Name: "Origin", Title: "Origin", Doc: "", Fields: []Field{ + {Key: "repo", Type: "string", Doc: "Repo is the repository, owner/name.", Nested: ""}, + {Key: "revision", Type: "string", Doc: "Revision is the commit the files were fetched at.", Nested: ""}, + {Key: "files", Type: "[]OriginFile", Doc: "Files are the weights fetched from it, with what the registry\npublished as each one's sha256. A new commit that leaves them alone —\na README edited — is not an update.", Nested: ""}, + {Key: "previous", Type: "*PreviousOrigin", Doc: "Previous is what the last update replaced, kept so it can be gone back\nto while its files are still on the mirror.", Nested: "catalog.PreviousOrigin"}, + }}, + {Package: "catalog", Name: "OriginFile", Title: "OriginFile", Doc: "", Fields: []Field{ + {Key: "path", Type: "string", Doc: "", Nested: ""}, + {Key: "sha256", Type: "string", Doc: "", Nested: ""}, + {Key: "bytes", Type: "int64", Doc: "", Nested: ""}, + }}, {Package: "config", Name: "Personal", Title: "Personal", Doc: "", Fields: []Field{ {Key: "disabled", Type: "bool", Doc: "Disabled refuses personal keys and personal chat; people then use the\ngateway only through projects they are members of.", Nested: ""}, {Key: "rpm", Type: "int", Doc: "", Nested: ""}, @@ -132,6 +145,11 @@ var Reference = []Block{ {Key: "balance_abs", Type: "float64", Doc: "", Nested: ""}, {Key: "balance_rel", Type: "float64", Doc: "", Nested: ""}, }}, + {Package: "catalog", Name: "PreviousOrigin", Title: "PreviousOrigin", Doc: "", Fields: []Field{ + {Key: "source", Type: "string", Doc: "", Nested: ""}, + {Key: "revision", Type: "string", Doc: "", Nested: ""}, + {Key: "files", Type: "[]OriginFile", Doc: "", Nested: ""}, + }}, {Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{ {Key: "name", Type: "string", Doc: "", Nested: ""}, {Key: "engine", Type: "string", Doc: "", Nested: ""},