diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc4ff8..468c583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,24 @@ APIs change between minors without a deprecation period. ### Added +- **Replicas counted per profile: seven on cards and four on CPUs, under one + name.** `llmgw ctl model replicas gpu=7 cpu=4`, the console's + Replicas per profile… and the mirror page's model card set how many of a + model's replicas run on each of its profiles, rather than every replica + on the best profile that fits. For a GGUF model with no profile on CPUs + one is added (`--cpu-memory`), preferred below the cards, so the router + fills the cards first and sends to the CPUs once they are full. A change + starts the replicas short before stopping those over: the model never + runs fewer. `ctl models` shows the counts; a scaling of another total + ends them. +- **Every setting of a served model from the mirror page.** A served + model's row has a Settings link, and its card now changes the context + and longest answer, the cards a replica takes, the replicas on each + profile (with the CPU's speed beside), stops and starts it, restarts its + replicas, and takes it out of the catalog with its files or without — + beside the replicas, class, owner, memory and adapters it held before. + Stopped on the row is now the model stopped and kept, started again as + it was, rather than its scaling set to nothing. - **How fast a model answers on the CPU, before it is served.** The serve form says, as the cores are changed, about how many tokens a second one reply gets on the CPU and on one card of the fleet's fastest class, and diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 4e525c3..3b949ab 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -418,7 +418,7 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n", name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m), - join(m.Aliases), join(m.Profiles), join(m.Fallbacks)) + join(m.Aliases), join(profilesWithCounts(m)), join(m.Fallbacks)) for _, x := range m.Warnings { warnings = append(warnings, m.Name+": "+x) } @@ -696,3 +696,16 @@ func reserves(m adminapi.ModelView) string { } return out } + +// profilesWithCounts are a model's profiles, each with its replicas when +// they are counted per profile: gpu=7, cpu=4. +func profilesWithCounts(m adminapi.ModelView) []string { + if len(m.ProfileReplicas) == 0 { + return m.Profiles + } + out := make([]string, len(m.Profiles)) + for i, p := range m.Profiles { + out[i] = fmt.Sprintf("%s=%d", p, m.ProfileReplicas[p]) + } + return out +} diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 0e424cb..8aad096 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -20,6 +20,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -406,6 +407,48 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog { }}) return nil }}, + {"Replicas per profile…", func(m *consoleModel) tea.Cmd { + var now []string + for _, p := range md.Profiles { + now = append(now, fmt.Sprintf("%s=%d", p, md.ProfileReplicas[p])) + } + m.openForm(&formDialog{title: "Replicas per profile of " + md.Name, + note: "How many replicas run on each profile, all answering under the model's name: seven on cards and " + + "four on CPUs, the router filling the cards first. A profile not named keeps its count; cpu=N with " + + "the memory below adds a profile on CPUs to a GGUF model that has none. Those short start first, " + + "then those over stop.", + fields: []formField{ + textF("replicas", strings.Join(now, " "), "profile=count, separated by spaces"), + textF("CPU memory GB", "", "only to add a profile on CPUs: what a replica reserves"), + textF("CPU cores", "", "of a replica on CPUs; empty is half the node's"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + req := adminapi.ProfileReplicasRequest{Profiles: map[string]int{}} + for _, f := range strings.Fields(v[0]) { + name, c, ok := strings.Cut(f, "=") + n, err := strconv.Atoi(c) + if !ok || err != nil || n < 0 || name == "" { + return nil, fmt.Errorf("%q: a profile and a count, as gpu=7", f) + } + req.Profiles[name] = n + } + if v[1] != "" { + mem, err := strconv.Atoi(v[1]) + if err != nil || mem < 1 { + return nil, fmt.Errorf("the CPU memory is a whole number of GB") + } + cores := 0 + if v[2] != "" { + if cores, err = strconv.Atoi(v[2]); err != nil || cores < 1 { + return nil, fmt.Errorf("the CPU cores are a whole number from 1") + } + } + req.CPU = &modelsvc.CPUProfile{MemoryGB: mem, CPUCores: cores} + } + return m.call("Replicas of "+md.Name, http.MethodPost, path+"/replicas", req, nil), nil + }}) + return nil + }}, {"Resources…", func(m *consoleModel) tea.Cmd { profiles := md.Profiles if len(profiles) == 0 { diff --git a/internal/control/adminapi/purge.go b/internal/control/adminapi/purge.go index c4021d8..67b4a39 100644 --- a/internal/control/adminapi/purge.go +++ b/internal/control/adminapi/purge.go @@ -2,6 +2,8 @@ package adminapi import ( "context" + "errors" + "fmt" "log/slog" "net/http" "net/url" @@ -48,10 +50,29 @@ type PurgeView struct { // purgeModel removes a model from the catalog and its files from the mirror // and the nodes. func (a *API) purgeModel(w http.ResponseWriter, r *http.Request, name string) { + out, err := a.Purge(r.Context(), name, actorOf(r)) + switch { + case errors.Is(err, errNoSuchModel): + writeError(w, http.StatusNotFound, err.Error()) + case err != nil: + writeError(w, http.StatusBadRequest, err.Error()) + default: + writeJSON(w, http.StatusOK, out) + } +} + +// errNoSuchModel is a model the catalog does not have. +var errNoSuchModel = errors.New("no such model in the catalog") + +// Purge removes a model from the catalog and its files from the mirror and +// the nodes, waiting a while for its replicas to stop before the nodes'. +func (a *API) Purge(ctx context.Context, name, by string) (PurgeView, error) { + if a.models == nil { + return PurgeView{}, errors.New("this control plane has no store") + } m, ok := a.ctl.Catalog().Resolve(name) if !ok { - writeError(w, http.StatusNotFound, "no model "+name+" in the catalog") - return + return PurgeView{}, fmt.Errorf("%w: %s", errNoSuchModel, name) } name = m.Name others := []catalog.Model{} @@ -60,37 +81,35 @@ func (a *API) purgeModel(w http.ResponseWriter, r *http.Request, name string) { others = append(others, o) } } - if err := a.models.Remove(r.Context(), name); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return + if err := a.models.Remove(ctx, name); err != nil { + return PurgeView{}, err } - a.presetDeclined(r.Context(), name, actorOf(r)) + a.presetDeclined(ctx, name, by) out := PurgeView{Removed: name} sources := purgeSources(m) if a.purgeMirror == nil { out.Note = "this control plane has no mirror; its files there, if any, are left" } else { - out.Mirror, out.Kept, out.Note = purgeFromMirror(r.Context(), a.purgeMirror, sources, others) + out.Mirror, out.Kept, out.Note = purgeFromMirror(ctx, a.purgeMirror, sources, others) } // The nodes' copies, once nothing runs on them. deadline := time.Now().Add(purgeWait) for a.runs(name) && time.Now().Before(deadline) { select { - case <-r.Context().Done(): - return + case <-ctx.Done(): + return out, ctx.Err() case <-time.After(time.Second): } } if a.runs(name) { out.Note = strings.TrimSpace(out.Note + " Its replicas had not stopped yet: its copies on the nodes are " + "left, and llmgw ctl weights --collect deletes them.") - writeJSON(w, http.StatusOK, out) - return + return out, nil } - out.Nodes = a.purgeFromNodes(r.Context(), sources, others) - writeJSON(w, http.StatusOK, out) + out.Nodes = a.purgeFromNodes(ctx, sources, others) + return out, nil } // runs says whether any replica of a model is still on a node. diff --git a/internal/control/adminapi/websettings.go b/internal/control/adminapi/websettings.go new file mode 100644 index 0000000..d3a4221 --- /dev/null +++ b/internal/control/adminapi/websettings.go @@ -0,0 +1,59 @@ +package adminapi + +import ( + "context" + "errors" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" +) + +// What the portal changes of a served model beside what the catalog holds: +// the same as the API's routes, called in the process. + +// SetModelContext changes a served model's context and longest answer. +func (a *API) SetModelContext(ctx context.Context, name string, req modelserve.ContextRequest, by string) (modelserve.ContextChange, error) { + if a.serve == nil { + return modelserve.ContextChange{}, errors.New("this control plane cannot change a model's context") + } + return a.serve.SetContext(ctx, name, req, by) +} + +// SetModelGPUs spreads a served model over more cards a replica, or fewer. +func (a *API) SetModelGPUs(ctx context.Context, name string, req modelserve.GPUsRequest, by string) (modelserve.GPUsChange, error) { + if a.serve == nil { + return modelserve.GPUsChange{}, errors.New("this control plane cannot change a model's cards") + } + return a.serve.SetGPUs(ctx, name, req, by) +} + +// RestartModel starts each of a model's ready replicas again, one at a time, +// and says how many. +func (a *API) RestartModel(name string) (int, error) { + var ids []string + for _, rep := range a.ctl.Reconciler().ReadyReplicas() { + if rep.Model == name { + ids = append(ids, rep.ID) + } + } + if len(ids) == 0 { + return 0, errors.New("none of its replicas is ready to restart") + } + return len(ids), a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...) +} + +// RemoveModel takes a model out of the catalog, with its files on the mirror +// and the nodes when purge. +func (a *API) RemoveModel(ctx context.Context, name string, purge bool, by string) (PurgeView, error) { + if purge { + return a.Purge(ctx, name, by) + } + if a.models == nil { + return PurgeView{}, errors.New("this control plane has no store") + } + if err := a.models.Remove(ctx, name); err != nil { + return PurgeView{}, err + } + a.presetDeclined(ctx, name, by) + return PurgeView{Removed: name}, nil +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index bac5b35..3d45014 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -108,6 +108,23 @@ type modelUse struct { // Declared models are read from server.yaml on every start, so a mode // changed here would revert; the page says where to change it instead. Declared bool + // What its card changes beside: the context and longest answer, whether + // it is stopped, its profile on cards and the cards a replica takes. + ContextLength, MaxOutput int + Stopped bool + GPUProfile string + GPUs int + // Profiles are where its replicas run and, Split, how many on each. + Profiles []profileUse + Split bool + // CanCPU says a profile on CPUs can be added — a llama.cpp model with + // none — reserving CPUMemoryGB and NewCPUCores, answering about + // CPUSpeed tokens a second on CPUSpeedNode. + CanCPU bool + CPUMemoryGB int + NewCPUCores int + CPUSpeed float64 + CPUSpeedNode string } // mirrorRow is one model version as the page shows it. @@ -324,6 +341,18 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { if name == "" || (mode != modeAlways && mode != modeOnDemand && mode != modeStopped) { return "", errors.New(l.T("mirror.err.choose_mode")) } + // Stopped is the model kept as it was, and started again so: + // not its scaling set to nothing, which forgot what it was — + // and its replicas per profile with it. + if mode == modeStopped { + if _, err := d.Catalog.Stop(ctx, name, admin.Email); err != nil { + return "", err + } + return l.T("mirror.msg.stopped", name), nil + } + if _, err := d.Catalog.Start(ctx, name, admin.Email); err == nil && mode == modeAlways { + return l.T("mirror.msg.model_started", name), nil + } replicas, _ := strconv.Atoi(r.FormValue("replicas")) sc := scalingFor(mode, replicas, 0) if _, err := d.Catalog.Scale(ctx, name, sc, admin.Email); err != nil { @@ -339,6 +368,7 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { })) mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel)) + p.registerModelSettings(mux, m) mux.HandleFunc("POST /admin/mirror/adapters", p.mirrorAction(m, p.setAdapters)) mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, @@ -565,6 +595,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, if len(m.Profiles) > 0 { u.MemoryGB, u.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores } + p.describeProfiles(&u, m, d.Rows[i].Status) if v, ok := views[m.Name]; ok { u.Fleet, u.Ready, u.Wanted = true, v.Ready, v.Desired u.Parked, u.Unplaceable = v.Parked, v.Unplaceable diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 6310b08..4f4102b 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -12,6 +12,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" @@ -34,6 +35,12 @@ type CatalogAdmin interface { SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) + // Stop takes a model out of service and keeps it; Start runs it again + // as it was. + Stop(ctx context.Context, name, by string) (catalog.Model, error) + Start(ctx context.Context, name, by string) (catalog.Model, error) + // SetProfileReplicas counts its replicas per profile, adding cpu. + SetProfileReplicas(ctx context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile, by string) (catalog.Model, error) } // serveForm is the offer to put one mirrored version into service. diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index 3eda1a4..b9f1519 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -12,6 +12,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" ) @@ -27,6 +28,22 @@ type fakeCatalog struct { owners []string resources []string adapterSets [][]catalog.Adapter + // stops, starts and splits are the models stopped, started and counted + // per profile, splitCPU the profile on CPUs last asked for. + stops, starts []string + splits []map[string]int + splitCPU *modelsvc.CPUProfile +} + +func (f *fakeCatalog) SetProfileReplicas(_ context.Context, name string, counts map[string]int, cpu *modelsvc.CPUProfile, + _ string, +) (catalog.Model, error) { + f.splits, f.splitCPU = append(f.splits, counts), cpu + m := catalog.Model{Name: name} + for p, n := range counts { + m.Profiles = append(m.Profiles, catalog.Profile{Name: p, Replicas: n}) + } + return m, f.err } func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { @@ -549,10 +566,12 @@ func TestTheCPUWithGPUsFreeIsAChoiceMadeOnPurpose(t *testing.T) { } func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, error) { + f.stops = append(f.stops, name) return catalog.Model{Name: name, Stopped: &catalog.Scaling{Min: 1, Max: 1}}, f.err } func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) { + f.starts = append(f.starts, name) return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err } diff --git a/internal/web/locales/en/mirror.json b/internal/web/locales/en/mirror.json index 18145dc..0318205 100644 --- a/internal/web/locales/en/mirror.json +++ b/internal/web/locales/en/mirror.json @@ -279,5 +279,44 @@ "strata.msg.served": "%s added: its first start fetches about %s GB onto the node and prepares it, which can take hours on a slow link; Replicas shows how far it is.", "strata.pack_files": "part of Strata's ready-made %s %s pack: served whole from the mirror's Strata tab, Strata fetching it onto the node", "strata.unknown_pack": "a pack made for Strata that Strata 0.2.7 does not know: its setup refuses it, and llama.cpp cannot read it", - "strata.old_card": "its engine needs an NVIDIA card of compute capability %s or newer (RTX 30 on); the nodes with Strata have older ones" + "strata.old_card": "its engine needs an NVIDIA card of compute capability %s or newer (RTX 30 on); the nodes with Strata have older ones", + "mirror.settings.open": "Settings", + "mirror.settings.heading": "Settings", + "mirror.settings.context": "Change the context", + "mirror.settings.context_note": "Leave one empty to keep it. The replicas start again on the new context, one at a time; a longest answer alone changes at once.", + "mirror.settings.gpus": "Cards a replica", + "mirror.settings.gpus_apply": "Spread it", + "mirror.settings.gpus_note": "Each replica moves to this many cards of its class, started again one at a time.", + "mirror.settings.start": "Start", + "mirror.settings.stop": "Stop", + "mirror.settings.stop_confirm": "Stop %s? Its replicas stop; it is kept, with its files, and Start runs it again as it was.", + "mirror.settings.restart": "Restart its replicas", + "mirror.settings.restart_confirm": "Start each of %s's replicas again, one at a time?", + "mirror.settings.unserve": "Remove from the catalog", + "mirror.settings.unserve_confirm": "Take %s out of the catalog? Its replicas stop; its files stay on the mirror and the nodes.", + "mirror.settings.purge": "Remove with its files", + "mirror.settings.purge_confirm": "Take %s out of the catalog and delete its files from the mirror and the nodes? What another model uses stays.", + "mirror.split.heading": "Replicas on each profile — all answer under the model's name; the cards are used first, the CPUs when they are full.", + "mirror.split.replicas": "replicas", + "mirror.split.on": "%d on %s", + "mirror.split.on_cpu": "On CPUs (%s)", + "mirror.split.on_gpu": "On cards (%s, %s)", + "mirror.split.add_cpu": "Also on CPUs", + "mirror.split.cpu_speed": "On the CPU about %s tokens a second for one reply, at best on %s.", + "mirror.split.note": "Those short start first, then those over stop: a change never runs fewer.", + "mirror.split.scaling_note": "Its replicas are counted per profile below; min and max are their sum.", + "mirror.msg.model_started": "%s is started again, as it was", + "mirror.msg.context": "%s: a context of %s tokens, answers of at most %s; %d replicas starting again", + "mirror.msg.gpus": "%s: %d cards a replica; %d replicas starting again", + "mirror.msg.restarting.one": "%[2]s: its replica is starting again", + "mirror.msg.restarting.other": "%[2]s: its %[1]d replicas are starting again, one at a time", + "mirror.msg.split": "%s runs %s", + "mirror.msg.unserved": "%s is out of the catalog; its files stay", + "mirror.msg.purged.one": "%[2]s is out of the catalog, its version removed from the mirror and its copies from the nodes.", + "mirror.msg.purged.other": "%[2]s is out of the catalog, %[1]d versions removed from the mirror and its copies from the nodes.", + "mirror.err.no_settings": "this control plane cannot change that", + "mirror.err.gpus": "the cards a replica takes is a whole number, 1 or more", + "mirror.err.action": "stop, start or restart", + "mirror.err.count": "%s is a whole number, 0 or more", + "mirror.err.choose_model": "choose a model" } diff --git a/internal/web/locales/tr/mirror.json b/internal/web/locales/tr/mirror.json index 90d2ce2..1754ae3 100644 --- a/internal/web/locales/tr/mirror.json +++ b/internal/web/locales/tr/mirror.json @@ -272,5 +272,42 @@ "strata.msg.served": "%s eklendi: ilk açılışında node'a yaklaşık %s GB indirip hazırlar; yavaş bağlantıda saatler sürebilir. İlerleyişi Replikalar gösterir.", "strata.pack_files": "Strata'nın hazır %s %s paketinin parçası: aynanın Strata sekmesinden bütün olarak sunulur, Strata onu node'a kendisi indirir", "strata.unknown_pack": "Strata için hazırlanmış ama Strata 0.2.7'nin tanımadığı bir paket: kurulumu onu reddeder, llama.cpp de okuyamaz", - "strata.old_card": "motoru compute capability %s ya da daha yeni bir NVIDIA kartı ister (RTX 30 ve sonrası); Strata kurulu node'ların kartları daha eski" + "strata.old_card": "motoru compute capability %s ya da daha yeni bir NVIDIA kartı ister (RTX 30 ve sonrası); Strata kurulu node'ların kartları daha eski", + "mirror.settings.open": "Ayarlar", + "mirror.settings.heading": "Ayarlar", + "mirror.settings.context": "Bağlamı değiştir", + "mirror.settings.context_note": "Değişmeyecek olanı boş bırakın. Replikalar yeni bağlamla teker teker yeniden başlar; yalnızca en uzun cevap değişirse hemen geçerli olur.", + "mirror.settings.gpus": "Replika başına kart", + "mirror.settings.gpus_apply": "Dağıt", + "mirror.settings.gpus_note": "Her replika kendi sınıfından bu kadar karta taşınır, teker teker yeniden başlatılarak.", + "mirror.settings.start": "Başlat", + "mirror.settings.stop": "Durdur", + "mirror.settings.stop_confirm": "%s durdurulsun mu? Replikaları kapanır; model dosyalarıyla birlikte saklanır, Başlat onu olduğu gibi yeniden çalıştırır.", + "mirror.settings.restart": "Replikaları yeniden başlat", + "mirror.settings.restart_confirm": "%s replikaları teker teker yeniden başlatılsın mı?", + "mirror.settings.unserve": "Katalogdan çıkar", + "mirror.settings.unserve_confirm": "%s katalogdan çıkarılsın mı? Replikaları kapanır; dosyaları aynada ve node'larda kalır.", + "mirror.settings.purge": "Dosyalarıyla birlikte sil", + "mirror.settings.purge_confirm": "%s katalogdan çıkarılıp dosyaları aynadan ve node'lardan silinsin mi? Başka bir modelin kullandıkları kalır.", + "mirror.split.heading": "Her profilde çalışan replika sayısı — hepsi modelin adıyla cevap verir; önce kartlar kullanılır, dolunca CPU'lar.", + "mirror.split.replicas": "replika sayısı", + "mirror.split.on": "%[2]s üzerinde %[1]d", + "mirror.split.on_cpu": "CPU'larda (%s)", + "mirror.split.on_gpu": "Kartlarda (%s, %s)", + "mirror.split.add_cpu": "CPU'larda da", + "mirror.split.cpu_speed": "CPU üzerinde tek bir cevap için yaklaşık %s token/sn, en iyi %s üzerinde.", + "mirror.split.note": "Önce eksik olanlar başlar, sonra fazlası kapanır: bir değişiklik sırasında daha az replika çalışmaz.", + "mirror.split.scaling_note": "Replikaları aşağıda profil başına sayılıyor; en az ve en çok bunların toplamıdır.", + "mirror.msg.model_started": "%s olduğu gibi yeniden başlatıldı", + "mirror.msg.context": "%s: %s token bağlam, en fazla %s token cevap; %d replika yeniden başlıyor", + "mirror.msg.gpus": "%s: replika başına %d kart; %d replika yeniden başlıyor", + "mirror.msg.restarting.other": "%[2]s: %[1]d replikası teker teker yeniden başlıyor", + "mirror.msg.split": "%s şöyle çalışıyor: %s", + "mirror.msg.unserved": "%s katalogdan çıkarıldı; dosyaları duruyor", + "mirror.msg.purged.other": "%[2]s katalogdan çıkarıldı, aynadan %[1]d sürümü ve node'lardaki kopyaları silindi.", + "mirror.err.no_settings": "bu control plane bunu değiştiremez", + "mirror.err.gpus": "replika başına kart sayısı 1 ya da daha büyük bir tam sayıdır", + "mirror.err.action": "durdur, başlat ya da yeniden başlat", + "mirror.err.count": "%s 0 ya da daha büyük bir tam sayıdır", + "mirror.err.choose_model": "bir model seçin" } diff --git a/internal/web/mirror_detail_test.go b/internal/web/mirror_detail_test.go index 196da04..e3aaf5e 100644 --- a/internal/web/mirror_detail_test.go +++ b/internal/web/mirror_detail_test.go @@ -2,14 +2,17 @@ package web import ( "context" + "fmt" "log/slog" "net/http" "net/url" + "slices" "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -333,3 +336,89 @@ func TestAnAdapterHeldInTheMirrorIsAttachedFromItsModelsCard(t *testing.T) { } } } + +// settingsFleet is roomyFleet that can change what the catalog does not +// hold of a model, and says what it was asked. +type settingsFleet struct { + roomyFleet + asked []string +} + +func (f *settingsFleet) SetModelContext(_ context.Context, name string, req modelserve.ContextRequest, _ string) (modelserve.ContextChange, error) { + f.asked = append(f.asked, fmt.Sprintf("context %s %d %d", name, req.Tokens, req.MaxOutput)) + return modelserve.ContextChange{Model: name, After: req.Tokens, MaxOutput: req.MaxOutput}, nil +} +func (f *settingsFleet) SetModelGPUs(_ context.Context, name string, req modelserve.GPUsRequest, _ string) (modelserve.GPUsChange, error) { + f.asked = append(f.asked, fmt.Sprintf("gpus %s %d", name, req.GPUs)) + return modelserve.GPUsChange{Model: name, After: req.GPUs}, nil +} +func (f *settingsFleet) RestartModel(name string) (int, error) { + f.asked = append(f.asked, "restart "+name) + return 1, nil +} +func (f *settingsFleet) RemoveModel(_ context.Context, name string, purge bool, _ string) (adminapi.PurgeView, error) { + f.asked = append(f.asked, fmt.Sprintf("remove %s %v", name, purge)) + return adminapi.PurgeView{Removed: name}, nil +} + +// A served model's row leads to its settings, and its card holds them all: +// context, cards, replicas per profile with CPUs to add, stop, restart, +// removal with its files or without. +func TestAServedModelsSettingsAreWhereItsFileIs(t *testing.T) { + cat := &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/coder/v1/c.gguf": { + {Name: "coder", ContextLength: 8192, MaxOutput: 1024, Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "gpu-1x6000-24g", Engine: "llamacpp", GPUClass: "6000-24g", GPUsPerNode: 1}}}}}} + fleet := &settingsFleet{roomyFleet: roomyFleet{more: 1}} + m := &fakeMirror{models: []mirror.Status{ + {Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 5 << 30}, + }} + mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: fleet}, true) + + if row := mirrorGet(t, mux, "/admin/mirror").Body.String(); !strings.Contains(row, `open=coder%3av1">Settings`) { + t.Error("the row of a served model does not lead to its settings") + } + card := mirrorGet(t, mux, "/admin/mirror?tab=held&open=coder:v1").Body.String() + for _, want := range []string{`action="/admin/mirror/context"`, `name="tokens" value="8192"`, `name="max_output" value="1024"`, + `action="/admin/mirror/gpus"`, `name="gpus" value="1"`, `action="/admin/mirror/split"`, `name="cpu_replicas"`, + `value="stop"`, `value="restart"`, `action="/admin/mirror/unserve"`, `name="purge" value="yes"`} { + if !strings.Contains(card, want) { + t.Errorf("the card has no %s", want) + } + } + + post := func(path string, kv ...string) string { + v := url.Values{"csrf": {"tok"}, "model": {"coder"}} + for i := 0; i+1 < len(kv); i += 2 { + v.Add(kv[i], kv[i+1]) + } + return mirrorPost(t, mux, path, v).Body.String() + } + post("/admin/mirror/context", "tokens", "16384", "max_output", "") + post("/admin/mirror/gpus", "gpus", "2") + post("/admin/mirror/state", "action", "restart") + post("/admin/mirror/unserve", "purge", "yes") + if want := []string{"context coder 16384 0", "gpus coder 2", "restart coder", "remove coder true"}; !slices.Equal(fleet.asked, want) { + t.Errorf("asked %v, want %v", fleet.asked, want) + } + post("/admin/mirror/state", "action", "stop") + post("/admin/mirror/state", "action", "start") + if !slices.Equal(cat.stops, []string{"coder"}) || !slices.Equal(cat.starts, []string{"coder"}) { + t.Errorf("stopped %v, started %v", cat.stops, cat.starts) + } + post("/admin/mirror/split", "profile", "gpu-1x6000-24g", "replicas_gpu-1x6000-24g", "7", + "cpu_replicas", "4", "cpu_memory", "12", "cpu_cores", "8") + if len(cat.splits) != 1 || cat.splits[0]["gpu-1x6000-24g"] != 7 || cat.splits[0]["cpu"] != 4 || + cat.splitCPU == nil || cat.splitCPU.MemoryGB != 12 || cat.splitCPU.CPUCores != 8 { + t.Errorf("split %v, cpu %+v", cat.splits, cat.splitCPU) + } +} + +// Stopped on the row is the model kept, not its scaling set to nothing. +func TestStoppingFromTheRowKeepsTheModel(t *testing.T) { + cat := coderCatalog() + mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{}, Catalog: cat}, true) + mirrorPost(t, mux, "/admin/mirror/mode", url.Values{"csrf": {"tok"}, "model": {"coder"}, "mode": {"stopped"}, "replicas": {"1"}}) + if !slices.Equal(cat.stops, []string{"coder"}) || len(cat.scaled) != 0 { + t.Errorf("stopped %v, scaled %v", cat.stops, cat.scaled) + } +} diff --git a/internal/web/mirror_settings.go b/internal/web/mirror_settings.go new file mode 100644 index 0000000..b33b133 --- /dev/null +++ b/internal/web/mirror_settings.go @@ -0,0 +1,237 @@ +package web + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// ModelSettings changes a served model beyond what the catalogue holds: its +// context, its cards, its replicas started again, its removal. The cluster +// view answers it where it can. +type ModelSettings interface { + SetModelContext(ctx context.Context, name string, req modelserve.ContextRequest, by string) (modelserve.ContextChange, error) + SetModelGPUs(ctx context.Context, name string, req modelserve.GPUsRequest, by string) (modelserve.GPUsChange, error) + RestartModel(name string) (int, error) + RemoveModel(ctx context.Context, name string, purge bool, by string) (adminapi.PurgeView, error) +} + +// settings is the cluster view's ModelSettings, nil where it has none. +func (p *AccountPages) settings() ModelSettings { + s, _ := p.fleet.(ModelSettings) + return s +} + +// registerModelSettings mounts what the card of a served model changes. +func (p *AccountPages) registerModelSettings(mux *http.ServeMux, m MirrorAdmin) { + need := func(r *http.Request) (string, ModelSettings, error) { + name := strings.TrimSpace(r.FormValue("model")) + if name == "" { + return "", nil, errors.New(langOf(r).T("mirror.err.choose_model")) + } + s := p.settings() + if s == nil { + return "", nil, errors.New(langOf(r).T("mirror.err.no_settings")) + } + return name, s, nil + } + + mux.HandleFunc("POST /admin/mirror/context", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + l := langOf(r) + name, s, err := need(r) + if err != nil { + return "", err + } + tokens, err := optionalCount(l, r.FormValue("tokens"), "serve.context_length") + if err != nil { + return "", err + } + maxOut, err := optionalCount(l, r.FormValue("max_output"), "serve.max_output") + if err != nil { + return "", err + } + ch, err := s.SetModelContext(ctx, name, modelserve.ContextRequest{Tokens: tokens, MaxOutput: maxOut}, admin.Email) + if err != nil { + return "", err + } + return l.T("mirror.msg.context", name, l.Int(int64(ch.After)), l.Int(int64(ch.MaxOutput)), ch.Restarted), nil + })) + + mux.HandleFunc("POST /admin/mirror/gpus", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + l := langOf(r) + name, s, err := need(r) + if err != nil { + return "", err + } + n, err := strconv.Atoi(strings.TrimSpace(r.FormValue("gpus"))) + if err != nil || n < 1 { + return "", errors.New(l.T("mirror.err.gpus")) + } + ch, err := s.SetModelGPUs(ctx, name, modelserve.GPUsRequest{GPUs: n, Profile: r.FormValue("profile")}, admin.Email) + if err != nil { + return "", err + } + return l.T("mirror.msg.gpus", name, ch.After, ch.Restarted), nil + })) + + mux.HandleFunc("POST /admin/mirror/state", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + l := langOf(r) + name := strings.TrimSpace(r.FormValue("model")) + switch r.FormValue("action") { + case "stop": + if _, err := p.catalog.Stop(ctx, name, admin.Email); err != nil { + return "", err + } + return l.T("mirror.msg.stopped", name), nil + case "start": + if _, err := p.catalog.Start(ctx, name, admin.Email); err != nil { + return "", err + } + return l.T("mirror.msg.model_started", name), nil + case "restart": + s := p.settings() + if s == nil { + return "", errors.New(l.T("mirror.err.no_settings")) + } + n, err := s.RestartModel(name) + if err != nil { + return "", err + } + return l.N("mirror.msg.restarting", int64(n), name), nil + } + return "", errors.New(l.T("mirror.err.action")) + })) + + mux.HandleFunc("POST /admin/mirror/split", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + l := langOf(r) + name := strings.TrimSpace(r.FormValue("model")) + counts := map[string]int{} + for _, prof := range r.Form["profile"] { + n, err := count(l, r.FormValue("replicas_"+prof), "mirror.split.replicas") + if err != nil { + return "", err + } + counts[prof] = n + } + var cpu *modelsvc.CPUProfile + if v := strings.TrimSpace(r.FormValue("cpu_replicas")); v != "" && v != "0" { + n, err := count(l, v, "mirror.split.replicas") + if err != nil { + return "", err + } + mem, err := count(l, r.FormValue("cpu_memory"), "serve.memory") + if err != nil { + return "", err + } + cores, err := optionalCount(l, r.FormValue("cpu_cores"), "serve.cpu_cores") + if err != nil { + return "", err + } + cpu = &modelsvc.CPUProfile{Name: "cpu", MemoryGB: mem, CPUCores: cores} + counts["cpu"] = n + } + got, err := p.catalog.SetProfileReplicas(ctx, name, counts, cpu, admin.Email) + if err != nil { + return "", err + } + var parts []string + for _, pr := range got.Profiles { + parts = append(parts, l.T("mirror.split.on", pr.Replicas, pr.Name)) + } + return l.T("mirror.msg.split", name, strings.Join(parts, ", ")), nil + })) + + mux.HandleFunc("POST /admin/mirror/unserve", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + l := langOf(r) + name, s, err := need(r) + if err != nil { + return "", err + } + purge := r.FormValue("purge") == "yes" + out, err := s.RemoveModel(ctx, name, purge, admin.Email) + if err != nil { + return "", err + } + if purge { + msg := l.N("mirror.msg.purged", int64(len(out.Mirror)), name) + if out.Note != "" { + msg += " " + out.Note + } + return msg, nil + } + return l.T("mirror.msg.unserved", name), nil + })) +} + +// count is a whole number of at least zero from a form field, or why not. +func count(l *Lang, v, field string) (int, error) { + n, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil || n < 0 { + return 0, errors.New(l.T("mirror.err.count", l.T(field))) + } + return n, nil +} + +// optionalCount is count, with an empty field zero: left as it is. +func optionalCount(l *Lang, v, field string) (int, error) { + if strings.TrimSpace(v) == "" { + return 0, nil + } + return count(l, v, field) +} + +// profileUse is one of a model's profiles as its card shows it. +type profileUse struct { + Name string + CPU bool + Class string + GPUs int + Replicas int +} + +// describeProfiles fills in what the card of a served model shows of its +// profiles, and of a profile on CPUs it could have. +func (p *AccountPages) describeProfiles(u *modelUse, m catalog.Model, st mirror.Status) { + u.ContextLength, u.MaxOutput, u.Stopped, u.Split = m.ContextLength, m.MaxOutput, m.Stopped != nil, m.Split() + hasCPU, llama := false, false + for _, pr := range m.Profiles { + pu := profileUse{Name: pr.Name, CPU: pr.CPUOnly(), Class: pr.GPUClass, Replicas: pr.Replicas, + GPUs: max(pr.GPUsPerNode, pr.TP*max(pr.PP, 1))} + u.Profiles = append(u.Profiles, pu) + hasCPU = hasCPU || pu.CPU + llama = llama || pr.Engine == "llamacpp" + if !pu.CPU && u.GPUProfile == "" && pr.GPU == nil { + u.GPUProfile, u.GPUs = pr.Name, max(pu.GPUs, 1) + } + } + if !llama || hasCPU || m.IsExternal() { + return + } + u.CanCPU = true + u.CPUMemoryGB = memoryFor(st.BytesTotal, st.BytesDone) + hosts := cpuHosts(p.fleet, u.CPUMemoryGB) + var cpus []int + for _, h := range hosts { + cpus = append(cpus, h.CPUs) + } + u.NewCPUCores = catalog.SuggestCores(cpus) + weights := st.BytesTotal + if weights <= 0 { + weights = st.BytesDone + } + u.CPUSpeed, u.CPUSpeedNode = bestCPUSpeed(hosts, weights, u.NewCPUCores) +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index da06349..a00f724 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -828,6 +828,14 @@ tr.held-detail > td { background: var(--bg); padding: .75rem 1rem 1rem; } .held-card .replicas-form .field-row { align-items: flex-end; } .held-card .replicas-form .field-row label { flex: 0 0 auto; } .held-card .replicas-form input[type=number] { width: 8rem; margin: .25rem 0 0; } +.held-card .model-settings { margin-top: .75rem; padding-top: .5rem; border-top: 1px dashed var(--line); } +.held-card .model-settings > summary { cursor: pointer; list-style: none; } +.held-card .model-settings > summary h4 { display: inline; font-size: .95rem; margin: 0; } +.held-card .settings-form { margin: .6rem 0; } +.held-card .settings-form .field-row { align-items: flex-end; } +.held-card .settings-form .field-row label { flex: 0 0 auto; } +.held-card .settings-form input[type=number] { width: 8rem; margin: .25rem 0 0; } +.held-card .settings-actions { display: flex; gap: .75rem; flex-wrap: wrap; align-items: center; margin-top: .6rem; } .held-card .replicas-form button { margin: 0; } a.open-held { text-decoration: none; color: inherit; } a.open-held:hover strong { text-decoration: underline; } diff --git a/internal/web/static/pages.js b/internal/web/static/pages.js index ee812c4..90204e9 100644 --- a/internal/web/static/pages.js +++ b/internal/web/static/pages.js @@ -15,7 +15,10 @@ } }); document.addEventListener('submit', function (e) { - var msg = e.target.getAttribute && e.target.getAttribute('data-confirm'); + // The button's own, where a form has several that mean different things. + var b = e.submitter; + var msg = (b && b.getAttribute && b.getAttribute('data-confirm')) || + (e.target.getAttribute && e.target.getAttribute('data-confirm')); if (msg && !window.confirm(msg)) { e.preventDefault(); } }); // A form being filled in is not refreshed away. The live parts of the diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 680091e..d98459c 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -109,6 +109,7 @@ {{else if eq (printf "%s" .State) "present"}} + {{if and .UsedBy (not .Detail)}}{{t "mirror.settings.open"}}{{end}} {{if and $.HasCatalog (not .UsedBy)}}{{if .Serve}}{{t "mirror.close"}} {{else}}{{t "mirror.serve"}}{{end}}{{end}}
+ +
-

{{if .MostNow}}{{if and .Headroom .Headroom.ByPreempting}}{{t "mirror.form.most_now_preempting" .MostNow}}{{else}}{{t "mirror.form.most_now" .MostNow}}{{end}} {{end}}{{t "mirror.form.note"}}

+

{{if .Split}}{{t "mirror.split.scaling_note"}} {{end}}{{if .MostNow}}{{if and .Headroom .Headroom.ByPreempting}}{{t "mirror.form.most_now_preempting" .MostNow}}{{else}}{{t "mirror.form.most_now" .MostNow}}{{end}} {{end}}{{t "mirror.form.note"}}

+{{template "mirror_model_settings" dict "M" . "Row" $row "CSRF" $csrf}} {{end}} {{end}} {{end}} + +{{/* The rest of what can be changed of a served model, where its file is: + its context, its cards, its replicas on each profile, stopped or + started, started again, and taken out of the catalog. */}} +{{define "mirror_model_settings"}}{{$m := .M}}{{$row := .Row}}{{$csrf := .CSRF}} +
+

{{t "mirror.settings.heading"}}

 + +
+ + + +
+ + + +
+

{{t "mirror.settings.context_note"}}

+
+ +{{if $m.GPUProfile}} +
+ + + + +
+ + +
+

{{t "mirror.settings.gpus_note"}}

+
+{{end}} + +{{if or (gt (len $m.Profiles) 1) $m.CanCPU}} +
+ + + +

{{t "mirror.split.heading"}}

+
+ {{range $m.Profiles}} + {{end}} + {{if $m.CanCPU}} + + {{end}} + +
+ {{if and $m.CanCPU $m.CPUSpeed}}

{{t "mirror.split.cpu_speed" (num $m.CPUSpeed 1) $m.CPUSpeedNode}}

{{end}} +

{{t "mirror.split.note"}}

+
+{{end}} + +
+
+ + + + {{if $m.Stopped}} + {{else}} + {{end}} +
+
+ + + +
+
+ + + + +
+
+
+{{end}}