Track the registry's new weights for served models, and move a model to them or back
A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a96b1a54b2
commit
44425193bd
15 files changed
+1393
-33
No files matched your search
@@ -35,7 +35,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client))
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
newModelRollbackCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
+31
-26
@@ -38,32 +38,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "job %s\n", job.ID)
|
||||
shown := map[string]string{}
|
||||
for {
|
||||
for _, s := range job.Steps {
|
||||
line := s.State + " " + s.Detail
|
||||
if shown[s.Name] == line {
|
||||
continue
|
||||
}
|
||||
shown[s.Name] = line
|
||||
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
||||
modelserve.Failed: "!!"}[s.State]
|
||||
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
||||
}
|
||||
switch job.State {
|
||||
case modelserve.Done:
|
||||
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
||||
return nil
|
||||
case modelserve.Failed:
|
||||
return errors.New(job.Error)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return followJob(cmd, client, job, "is served")
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
@@ -79,3 +54,33 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
||||
return c
|
||||
}
|
||||
|
||||
// followJob shows a job's steps as they happen, until it ends.
|
||||
func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, doneSays string) error {
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "job %s\n", job.ID)
|
||||
shown := map[string]string{}
|
||||
for {
|
||||
for _, s := range job.Steps {
|
||||
line := s.State + " " + s.Detail
|
||||
if shown[s.Name] == line {
|
||||
continue
|
||||
}
|
||||
shown[s.Name] = line
|
||||
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
||||
modelserve.Failed: "!!", modelserve.Skipped: "--"}[s.State]
|
||||
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
||||
}
|
||||
switch job.State {
|
||||
case modelserve.Done:
|
||||
fmt.Fprintf(out, "\n%s %s.\n", job.Model, doneSays)
|
||||
return nil
|
||||
case modelserve.Failed:
|
||||
return errors.New(job.Error)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
func newModelUpdatesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var check bool
|
||||
c := &cobra.Command{
|
||||
Use: "updates [MODEL]",
|
||||
Short: "Show which models' repositories have published new weights",
|
||||
Long: "A model served from the registry remembers the repository and commit its\n" +
|
||||
"files came from, and the control plane asks the registry every\n" +
|
||||
"mirror.update_check (six hours by default) whether those files have\n" +
|
||||
"changed there. A new commit that leaves them alone is not an update.\n\n" +
|
||||
"--check asks now rather than showing the last answer.",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var list []modelserve.UpdateInfo
|
||||
if check || len(args) == 1 {
|
||||
req := map[string]string{}
|
||||
if len(args) == 1 {
|
||||
req["model"] = args[0]
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
if err := client().post(cmd.Context(), "/api/v1/updates/check", body, &list); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := client().get(cmd.Context(), "/api/v1/updates", &list); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(list)
|
||||
}
|
||||
if len(list) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "no model has an origin to compare yet; one is checked a few minutes after the control plane starts")
|
||||
return nil
|
||||
}
|
||||
w := tabTo(cmd.OutOrStdout())
|
||||
fmt.Fprintln(w, "MODEL\tREPOSITORY\tRUNS\tLATEST\tUPDATE\tCHECKED")
|
||||
for _, u := range list {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", u.Model, u.Repo, catalog.ShortRevision(u.Current),
|
||||
catalog.ShortRevision(u.Latest), updateCell(u), ago(u.CheckedAt, time.Now()))
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, u := range list {
|
||||
if u.Available {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\nTo move one to its new weights: llmgw ctl model update MODEL\n")
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&check, "check", false, "ask the registry now")
|
||||
return c
|
||||
}
|
||||
|
||||
// updateCell says what an update would be, or why there is none.
|
||||
func updateCell(u modelserve.UpdateInfo) string {
|
||||
switch {
|
||||
case u.Err != "":
|
||||
return "could not ask: " + truncate(u.Err, 60)
|
||||
case u.Available:
|
||||
return fmt.Sprintf("yes: %d file(s), %s", len(u.Changed), bytesShort(u.Bytes))
|
||||
case len(u.Missing) > 0:
|
||||
return "the repository no longer has " + strings.Join(u.Missing, ", ")
|
||||
case u.Latest != "" && u.Latest != u.Current:
|
||||
return "no: new commit, same weights"
|
||||
}
|
||||
return "-"
|
||||
}
|
||||
|
||||
func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "update MODEL",
|
||||
Short: "Move a model to the weights its repository publishes now",
|
||||
Long: "The new files are fetched into the mirror beside the ones in use and\n" +
|
||||
"checked against the registry's checksums and the nodes' llama.cpp; then\n" +
|
||||
"the catalog names them and the replicas are replaced one at a time, each\n" +
|
||||
"new one up before an old one stops, and one is asked to say hello.\n\n" +
|
||||
"The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" +
|
||||
"back to them. A model from server.yaml is changed there instead.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil {
|
||||
return err
|
||||
}
|
||||
return followJob(cmd, client, job, "runs the new weights")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newModelRollbackCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "rollback MODEL",
|
||||
Short: "Move a model back to the weights its last update replaced",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var out map[string]string
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/rollback", []byte("{}"), &out); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), out["result"])
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -128,6 +128,7 @@ func newServerCmd() *cobra.Command {
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
updateMetrics := &modelserve.UpdateCollector{}
|
||||
// The store's cluster, when it is one; read by the metrics and
|
||||
// by `ctl status`.
|
||||
var storeHA *pgha.Collector
|
||||
@@ -183,6 +184,11 @@ func newServerCmd() *cobra.Command {
|
||||
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Whether models' repositories have new weights; filled in
|
||||
// once the mirror is set up, below.
|
||||
if err := rt.RegisterMetrics(updateMetrics); err != nil {
|
||||
return err
|
||||
}
|
||||
// A store of several hosts is a replicated cluster: what state
|
||||
// it is in is published beside everything else, read from its
|
||||
// Patroni. One host is a single PostgreSQL, which has none.
|
||||
@@ -321,10 +327,17 @@ func newServerCmd() *cobra.Command {
|
||||
// One command from a registry repository to a replica that
|
||||
// answers: `llmgw ctl model serve`.
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||
serve := modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
}))
|
||||
// Updates: the catalog in force, the service that
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart,
|
||||
})
|
||||
adminAPI.SetServe(serve)
|
||||
updateMetrics.Set(serve)
|
||||
go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck)
|
||||
}
|
||||
log.Info("model mirror configured",
|
||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
@@ -418,6 +419,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
})
|
||||
}},
|
||||
}
|
||||
items = append(items, m.updateItems(md)...)
|
||||
if md.Parked {
|
||||
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
|
||||
@@ -674,3 +676,57 @@ func (m *consoleModel) doEntry() tea.Cmd {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateItems are what can be done about a model's repository publishing new
|
||||
// weights: move to them, ask now, or go back.
|
||||
func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||||
if md.Origin == nil {
|
||||
return nil
|
||||
}
|
||||
server := "--server=" + m.client.base
|
||||
var items []menuItem
|
||||
if u := md.Update; u != nil && u.Available {
|
||||
items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)),
|
||||
func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Update " + md.Name,
|
||||
text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+
|
||||
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
|
||||
"replicas are replaced one at a time. The old files stay, to go back to.",
|
||||
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")),
|
||||
yes: func(m *consoleModel) tea.Cmd { return m.runAfter("ctl", server, "model", "update", md.Name) }}
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items, menuItem{"Ask the registry for new weights now", func(m *consoleModel) tea.Cmd {
|
||||
return m.call("New weights for "+md.Name, http.MethodPost, "/api/v1/updates/check", map[string]string{"model": md.Name},
|
||||
func(raw json.RawMessage) string {
|
||||
var us []modelserve.UpdateInfo
|
||||
if json.Unmarshal(raw, &us) != nil || len(us) == 0 {
|
||||
return string(raw)
|
||||
}
|
||||
u := us[0]
|
||||
if u.Available {
|
||||
return fmt.Sprintf("%s has new weights at %s: %s, %s.", u.Repo, catalog.ShortRevision(u.Latest),
|
||||
strings.Join(u.Changed, ", "), bytesShort(u.Bytes))
|
||||
}
|
||||
return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u))
|
||||
})
|
||||
}})
|
||||
if prev := md.Origin.Previous; prev != "" {
|
||||
items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Go back on " + md.Name,
|
||||
text: "It runs the weights its last update replaced, " + catalog.ShortRevision(prev) +
|
||||
", if they are still on the mirror;\nits replicas are replaced one at a time.",
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Go back on "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/rollback",
|
||||
map[string]string{}, func(raw json.RawMessage) string {
|
||||
var out map[string]string
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
return out["result"] + "."
|
||||
})
|
||||
}}
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
return items
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
@@ -959,13 +960,17 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
if len(md.Warnings) > 0 {
|
||||
warn = sWarn.Render("!")
|
||||
}
|
||||
upd := ""
|
||||
if md.Update != nil && md.Update.Available {
|
||||
upd = sWarn.Render("↑ " + catalog.ShortRevision(md.Update.Latest))
|
||||
}
|
||||
rows = append(rows, []string{md.Name, fmt.Sprintf("%d/%d", md.Ready, md.Desired), status, md.Class,
|
||||
strings.Join(md.Profiles, ","), warn})
|
||||
strings.Join(md.Profiles, ","), upd, warn})
|
||||
}
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter actions · A add from a file · ← back · Q leave"
|
||||
}
|
||||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", ""}, rows, m.row, pw, m.inPanel), keys
|
||||
return "Models", "\n" + table([]string{"MODEL", "READY", "STATUS", "CLASS", "PROFILES", "UPDATE", ""}, rows, m.row, pw, m.inPanel), keys
|
||||
case cReplicas:
|
||||
var rows [][]string
|
||||
for _, r := range m.data.replicas {
|
||||
@@ -1174,6 +1179,16 @@ func (m consoleModel) modelDetail(md adminapi.ModelView) string {
|
||||
kv("profiles", strings.Join(md.Profiles, ", "))
|
||||
kv("aliases", strings.Join(md.Aliases, ", "))
|
||||
kv("source", md.Source)
|
||||
if o := md.Origin; o != nil {
|
||||
from := o.Repo + " at " + catalog.ShortRevision(o.Revision)
|
||||
if o.Previous != "" {
|
||||
from += ", updated from " + catalog.ShortRevision(o.Previous)
|
||||
}
|
||||
kv("from", from)
|
||||
}
|
||||
if u := md.Update; u != nil {
|
||||
kv("registry", updateCell(*u)+sDim.Render(", asked "+ago(u.CheckedAt, time.Now())))
|
||||
}
|
||||
if md.MemoryGB > 0 {
|
||||
kv("reserves", fmt.Sprintf("%d GB, %d cores (uses %.1f GB)", md.MemoryGB, md.CPUCores, md.MeasuredMemoryGB))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user