diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a0a303..459ca0a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,14 @@ organization's usage over time by team, project or person, its teams and projects with limits and spend, and its members — read-only, and only what was spent in its projects. Administrators can open every organization's page. +### How a model runs, from the mirror page + +Each model a mirrored file is serving now shows how it runs — always on, on +demand, or stopped — and it can be changed there, without the CLI. The serve +form asks the same: on demand loads the model when somebody asks for it and +stops it after 15 minutes unused. A model declared in `server.yaml` shows its +mode and says to change it there, because the file is read on every start. + ### Fixes - Usage by team, project or organization, and the spend on the quotas page, diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index b3c94dce..f9024d6a 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -71,16 +71,16 @@ func (s *Service) Declared(name string) bool { } // Sources maps each weights source in the catalog to the models that name it, -// so a page listing files can say which of them are in use. -func (s *Service) Sources(ctx context.Context) (map[string][]string, error) { +// so a page listing files can say which of them are in use and how they run. +func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, error) { c, err := s.Load(ctx) if err != nil { return nil, err } - out := map[string][]string{} + out := map[string][]catalog.Model{} for _, m := range c.Models() { if m.Source != "" { - out[m.Source] = append(out[m.Source], m.Name) + out[m.Source] = append(out[m.Source], m) } } return out, nil diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index c3379b74..5ed44344 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "net/http" "sort" "strconv" @@ -54,6 +55,16 @@ type repoSet struct { type repoSource struct{ URL, Path string } +// modelUse is a catalog model that runs a mirrored version, and how. +type modelUse struct { + Name string + Mode string // always, ondemand or stopped + Replicas int + // Declared models are read from server.yaml on every start, so a mode + // changed here would revert; the page says where to change it instead. + Declared bool +} + // mirrorRow is one model version as the page shows it. type mirrorRow struct { mirror.Status @@ -75,9 +86,10 @@ type mirrorRow struct { // row — a fifteen-shard model is one thing to serve or remove, and fifteen // rows of it pushed everything else off the page. Files int - // UsedBy names the catalog models whose source is this row's address, so - // a file that is in use does not look like one waiting to be served. - UsedBy []string + // UsedBy is the catalog models whose source is this row's address, and how + // each runs, so a file in use does not look like one waiting to be served + // and how it runs can be changed where it is seen. + UsedBy []modelUse } type adminMirrorData struct { @@ -193,6 +205,27 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { })) if d.Catalog != nil { + // How a model runs, changed where the file it runs is seen. + mux.HandleFunc("POST /admin/mirror/mode", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + name, mode := r.FormValue("model"), r.FormValue("mode") + if name == "" || (mode != modeAlways && mode != modeOnDemand && mode != modeStopped) { + return "", errors.New("choose a model and how it should run") + } + replicas, _ := strconv.Atoi(r.FormValue("replicas")) + sc := scalingFor(mode, replicas, 0) + if _, err := d.Catalog.Scale(ctx, name, sc, admin.Email); err != nil { + return "", err + } + switch mode { + case modeOnDemand: + return name + " now loads when asked for and stops after 15 minutes unused", nil + case modeStopped: + return name + " is stopped; its replicas are being taken down", nil + } + return name + " now runs all the time", nil + })) + mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { model, err := buildModel(r.FormValue) @@ -286,7 +319,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, } d.Base = got.Base - var sources map[string][]string + var sources map[string][]catalog.Model if p.catalog != nil { if sources, err = p.catalog.Sources(r.Context()); err != nil { p.log.Warn("could not read which models use the mirror", "err", err) @@ -324,7 +357,12 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, d.Rows = append(d.Rows, row) } for i := range d.Rows { - d.Rows[i].UsedBy = sources[d.Rows[i].URL] + for _, m := range sources[d.Rows[i].URL] { + d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, modelUse{ + Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1), + Declared: p.catalog.Declared(m.Name), + }) + } } // The one row whose serve form is open, and only when the catalog can be diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index c5dc6ade..1716461f 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -3,6 +3,7 @@ package web import ( "context" "errors" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "io" "log/slog" "net/http" @@ -542,7 +543,8 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) { {Model: "big", Version: "v1", File: "a-00002-of-00002.gguf", State: mirror.Present, BytesDone: 1 << 30}, {Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20}, }} - cat := &fakeCatalog{sources: map[string][]string{"http://mirror:8081/small/v1/s.gguf": {"small-chat"}}} + cat := &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": { + {Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}} body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true), "/admin/mirror").Body.String() @@ -567,3 +569,32 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) { t.Error("a file nobody uses should be offered for serving") } } + +func TestHowAModelRunsIsChangedWhereItsFileIsShown(t *testing.T) { + m := &fakeMirror{models: []mirror.Status{ + {Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20}, + }} + cat := &fakeCatalog{declared: "fixed", sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": { + {Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}}, + {Name: "fixed", Scaling: catalog.Scaling{Min: 0, Max: 1}}, + }}} + mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true) + + body := mirrorGet(t, mux, "/admin/mirror").Body.String() + if !strings.Contains(body, ``) { + t.Error("the running model's mode is not shown as chosen") + } + // Declared in server.yaml: shown, not offered, since it would revert. + if !strings.Contains(body, "on demand · set in server.yaml") { + t.Error("a declared model should say where its mode is set") + } + if strings.Count(body, `name="mode" class="mode"`) != 1 { + t.Error("only the runtime model should have a mode selector") + } + + mirrorPost(t, mux, "/admin/mirror/mode", url.Values{"csrf": {"tok"}, + "model": {"small-chat"}, "mode": {"ondemand"}, "replicas": {"1"}}) + if len(cat.scaled) != 1 || cat.lastScale.Min != 0 || cat.lastScale.Max != 1 || cat.lastScale.IdleTimeout == 0 { + t.Errorf("on demand was sent as %+v", cat.lastScale) + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index aadb34e5..74371c08 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "strings" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" @@ -16,7 +17,10 @@ type CatalogAdmin interface { Add(ctx context.Context, m catalog.Model, by string) error Declared(name string) bool // Sources maps a weights source to the catalog models that use it. - Sources(ctx context.Context) (map[string][]string, error) + Sources(ctx context.Context) (map[string][]catalog.Model, error) + // Scale changes how a model added at runtime runs: always, on demand, or + // not at all. One declared in configuration is refused. + Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) } // serveForm is the offer to put one mirrored version into service. @@ -211,7 +215,7 @@ func buildModel(get func(string) string) (catalog.Model, error) { if err != nil { return m, err } - m.Scaling = catalog.Scaling{Min: min, Max: min} + m.Scaling = scalingFor(get("mode"), min, 0) engine := strings.TrimSpace(get("engine")) if engine == "" { @@ -300,3 +304,45 @@ func sep(existing string) string { } return ", and " } + +// Run modes a page offers for a model, in the words a person would use. +const ( + modeAlways = "always" + modeOnDemand = "ondemand" + modeStopped = "stopped" +) + +// defaultIdle is how long an on-demand model set from a page stays loaded +// after its last request. +const defaultIdle = 15 * time.Minute + +// scalingFor turns a run mode into scaling: replicas is how many run, or may +// run on demand; idle keeps an on-demand model's timeout when it has one. +func scalingFor(mode string, replicas int, idle time.Duration) catalog.Scaling { + if replicas < 1 { + replicas = 1 + } + switch mode { + case modeOnDemand: + if idle <= 0 { + idle = defaultIdle + } + return catalog.Scaling{Min: 0, Max: replicas, IdleTimeout: idle} + case modeStopped: + return catalog.Scaling{Min: 0, Max: 0} + default: + return catalog.Scaling{Min: replicas, Max: replicas} + } +} + +// modeOf names how a model runs. +func modeOf(sc catalog.Scaling) string { + switch { + case sc.Min > 0: + return modeAlways + case sc.Max > 0: + return modeOnDemand + default: + return modeStopped + } +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index 2c12501a..40210bf7 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -15,10 +15,12 @@ import ( ) type fakeCatalog struct { - added []catalog.Model - declared string - sources map[string][]string - err error + added []catalog.Model + declared string + sources map[string][]catalog.Model + scaled []string + lastScale catalog.Scaling + err error } func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { @@ -29,10 +31,16 @@ func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { return nil } func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } -func (f *fakeCatalog) Sources(context.Context) (map[string][]string, error) { +func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, error) { return f.sources, nil } +func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) { + f.scaled = append(f.scaled, name) + f.lastScale = sc + return catalog.Model{Name: name, Scaling: sc}, f.err +} + // fleet reports a cluster with a given number of cards and engines. type fakeFleet struct { gpus int @@ -405,3 +413,17 @@ func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) { t.Errorf("the server's answer should be shown; got %s", body) } } + +func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) { + cat := &fakeCatalog{} + mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"big"}, "source": {reachableSource(t)}, "engine": {"llamacpp"}, + "replicas": {"1"}, "mode": {"ondemand"}, "context_length": {"4096"}, "profile": {"cpu"}, + }) + if len(cat.added) != 1 { + t.Fatalf("expected one model added, got %d", len(cat.added)) + } + if sc := cat.added[0].Scaling; sc.Min != 0 || sc.Max != 1 || sc.IdleTimeout == 0 { + t.Errorf("on demand was added as %+v", sc) + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 6559d38e..1781add6 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -445,3 +445,5 @@ h3.limits-for { font-size: .92rem; margin: 1rem 0 .4rem; } table.narrow-table { max-width: 30rem; } .q-act a + a { margin-left: .8rem; } .chip-role { font-size: .68rem; font-weight: 600; color: var(--accent); text-transform: uppercase; letter-spacing: .04em; } +table.held .use { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap; margin: .1rem 0; } +table.held select.mode { padding: .1rem .3rem; font-size: .8rem; } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 5808d04f..4d825bd3 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -49,7 +49,25 @@ reachable from this machine.