From bf94cd16671c9f7357452c3145d9aa25f0588768 Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Mon, 28 Sep 2026 20:34:41 +0200
Subject: [PATCH] Set how a model runs from the mirror page
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Every mirrored version was in use on the cluster, so the serve link — hidden for a version already served — appeared nowhere, and changing a model between always on and on demand needed the CLI. Each model using a file now shows how it runs and offers always on, on demand (15 minutes idle) or stopped, through the model service's Scale; models declared in server.yaml show their mode and where to change it. The serve form asks the same question.
Co-Authored-By: Claude Opus 5.5 (1M context)
---
CHANGELOG.md | 8 ++++
internal/control/modelsvc/modelsvc.go | 8 ++--
internal/web/admin_mirror.go | 48 ++++++++++++++++++++---
internal/web/admin_mirror_test.go | 33 +++++++++++++++-
internal/web/admin_serve.go | 50 +++++++++++++++++++++++-
internal/web/admin_serve_test.go | 32 ++++++++++++---
internal/web/static/base.css | 2 +
internal/web/templates/admin_mirror.html | 25 +++++++++++-
8 files changed, 188 insertions(+), 18 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 78a0a303..459ca0a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,14 @@ organization's usage over time by team, project or person, its teams and
projects with limits and spend, and its members — read-only, and only what was
spent in its projects. Administrators can open every organization's page.
+### How a model runs, from the mirror page
+
+Each model a mirrored file is serving now shows how it runs — always on, on
+demand, or stopped — and it can be changed there, without the CLI. The serve
+form asks the same: on demand loads the model when somebody asks for it and
+stops it after 15 minutes unused. A model declared in `server.yaml` shows its
+mode and says to change it there, because the file is read on every start.
+
### Fixes
- Usage by team, project or organization, and the spend on the quotas page,
diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go
index b3c94dce..f9024d6a 100644
--- a/internal/control/modelsvc/modelsvc.go
+++ b/internal/control/modelsvc/modelsvc.go
@@ -71,16 +71,16 @@ func (s *Service) Declared(name string) bool {
}
// Sources maps each weights source in the catalog to the models that name it,
-// so a page listing files can say which of them are in use.
-func (s *Service) Sources(ctx context.Context) (map[string][]string, error) {
+// so a page listing files can say which of them are in use and how they run.
+func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, error) {
c, err := s.Load(ctx)
if err != nil {
return nil, err
}
- out := map[string][]string{}
+ out := map[string][]catalog.Model{}
for _, m := range c.Models() {
if m.Source != "" {
- out[m.Source] = append(out[m.Source], m.Name)
+ out[m.Source] = append(out[m.Source], m)
}
}
return out, nil
diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go
index c3379b74..5ed44344 100644
--- a/internal/web/admin_mirror.go
+++ b/internal/web/admin_mirror.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"net/http"
"sort"
"strconv"
@@ -54,6 +55,16 @@ type repoSet struct {
type repoSource struct{ URL, Path string }
+// modelUse is a catalog model that runs a mirrored version, and how.
+type modelUse struct {
+ Name string
+ Mode string // always, ondemand or stopped
+ Replicas int
+ // Declared models are read from server.yaml on every start, so a mode
+ // changed here would revert; the page says where to change it instead.
+ Declared bool
+}
+
// mirrorRow is one model version as the page shows it.
type mirrorRow struct {
mirror.Status
@@ -75,9 +86,10 @@ type mirrorRow struct {
// row — a fifteen-shard model is one thing to serve or remove, and fifteen
// rows of it pushed everything else off the page.
Files int
- // UsedBy names the catalog models whose source is this row's address, so
- // a file that is in use does not look like one waiting to be served.
- UsedBy []string
+ // UsedBy is the catalog models whose source is this row's address, and how
+ // each runs, so a file in use does not look like one waiting to be served
+ // and how it runs can be changed where it is seen.
+ UsedBy []modelUse
}
type adminMirrorData struct {
@@ -193,6 +205,27 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
}))
if d.Catalog != nil {
+ // How a model runs, changed where the file it runs is seen.
+ mux.HandleFunc("POST /admin/mirror/mode", p.mirrorAction(m,
+ func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
+ name, mode := r.FormValue("model"), r.FormValue("mode")
+ if name == "" || (mode != modeAlways && mode != modeOnDemand && mode != modeStopped) {
+ return "", errors.New("choose a model and how it should run")
+ }
+ replicas, _ := strconv.Atoi(r.FormValue("replicas"))
+ sc := scalingFor(mode, replicas, 0)
+ if _, err := d.Catalog.Scale(ctx, name, sc, admin.Email); err != nil {
+ return "", err
+ }
+ switch mode {
+ case modeOnDemand:
+ return name + " now loads when asked for and stops after 15 minutes unused", nil
+ case modeStopped:
+ return name + " is stopped; its replicas are being taken down", nil
+ }
+ return name + " now runs all the time", nil
+ }))
+
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
model, err := buildModel(r.FormValue)
@@ -286,7 +319,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
}
d.Base = got.Base
- var sources map[string][]string
+ var sources map[string][]catalog.Model
if p.catalog != nil {
if sources, err = p.catalog.Sources(r.Context()); err != nil {
p.log.Warn("could not read which models use the mirror", "err", err)
@@ -324,7 +357,12 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
d.Rows = append(d.Rows, row)
}
for i := range d.Rows {
- d.Rows[i].UsedBy = sources[d.Rows[i].URL]
+ for _, m := range sources[d.Rows[i].URL] {
+ d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, modelUse{
+ Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
+ Declared: p.catalog.Declared(m.Name),
+ })
+ }
}
// The one row whose serve form is open, and only when the catalog can be
diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go
index c5dc6ade..1716461f 100644
--- a/internal/web/admin_mirror_test.go
+++ b/internal/web/admin_mirror_test.go
@@ -3,6 +3,7 @@ package web
import (
"context"
"errors"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"io"
"log/slog"
"net/http"
@@ -542,7 +543,8 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) {
{Model: "big", Version: "v1", File: "a-00002-of-00002.gguf", State: mirror.Present, BytesDone: 1 << 30},
{Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
}}
- cat := &fakeCatalog{sources: map[string][]string{"http://mirror:8081/small/v1/s.gguf": {"small-chat"}}}
+ cat := &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
+ {Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}}
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true),
"/admin/mirror").Body.String()
@@ -567,3 +569,32 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) {
t.Error("a file nobody uses should be offered for serving")
}
}
+
+func TestHowAModelRunsIsChangedWhereItsFileIsShown(t *testing.T) {
+ m := &fakeMirror{models: []mirror.Status{
+ {Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
+ }}
+ cat := &fakeCatalog{declared: "fixed", sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
+ {Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}},
+ {Name: "fixed", Scaling: catalog.Scaling{Min: 0, Max: 1}},
+ }}}
+ mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true)
+
+ body := mirrorGet(t, mux, "/admin/mirror").Body.String()
+ if !strings.Contains(body, ``) {
+ t.Error("the running model's mode is not shown as chosen")
+ }
+ // Declared in server.yaml: shown, not offered, since it would revert.
+ if !strings.Contains(body, "on demand · set in server.yaml") {
+ t.Error("a declared model should say where its mode is set")
+ }
+ if strings.Count(body, `name="mode" class="mode"`) != 1 {
+ t.Error("only the runtime model should have a mode selector")
+ }
+
+ mirrorPost(t, mux, "/admin/mirror/mode", url.Values{"csrf": {"tok"},
+ "model": {"small-chat"}, "mode": {"ondemand"}, "replicas": {"1"}})
+ if len(cat.scaled) != 1 || cat.lastScale.Min != 0 || cat.lastScale.Max != 1 || cat.lastScale.IdleTimeout == 0 {
+ t.Errorf("on demand was sent as %+v", cat.lastScale)
+ }
+}
diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go
index aadb34e5..74371c08 100644
--- a/internal/web/admin_serve.go
+++ b/internal/web/admin_serve.go
@@ -6,6 +6,7 @@ import (
"fmt"
"strconv"
"strings"
+ "time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
@@ -16,7 +17,10 @@ type CatalogAdmin interface {
Add(ctx context.Context, m catalog.Model, by string) error
Declared(name string) bool
// Sources maps a weights source to the catalog models that use it.
- Sources(ctx context.Context) (map[string][]string, error)
+ Sources(ctx context.Context) (map[string][]catalog.Model, error)
+ // Scale changes how a model added at runtime runs: always, on demand, or
+ // not at all. One declared in configuration is refused.
+ Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error)
}
// serveForm is the offer to put one mirrored version into service.
@@ -211,7 +215,7 @@ func buildModel(get func(string) string) (catalog.Model, error) {
if err != nil {
return m, err
}
- m.Scaling = catalog.Scaling{Min: min, Max: min}
+ m.Scaling = scalingFor(get("mode"), min, 0)
engine := strings.TrimSpace(get("engine"))
if engine == "" {
@@ -300,3 +304,45 @@ func sep(existing string) string {
}
return ", and "
}
+
+// Run modes a page offers for a model, in the words a person would use.
+const (
+ modeAlways = "always"
+ modeOnDemand = "ondemand"
+ modeStopped = "stopped"
+)
+
+// defaultIdle is how long an on-demand model set from a page stays loaded
+// after its last request.
+const defaultIdle = 15 * time.Minute
+
+// scalingFor turns a run mode into scaling: replicas is how many run, or may
+// run on demand; idle keeps an on-demand model's timeout when it has one.
+func scalingFor(mode string, replicas int, idle time.Duration) catalog.Scaling {
+ if replicas < 1 {
+ replicas = 1
+ }
+ switch mode {
+ case modeOnDemand:
+ if idle <= 0 {
+ idle = defaultIdle
+ }
+ return catalog.Scaling{Min: 0, Max: replicas, IdleTimeout: idle}
+ case modeStopped:
+ return catalog.Scaling{Min: 0, Max: 0}
+ default:
+ return catalog.Scaling{Min: replicas, Max: replicas}
+ }
+}
+
+// modeOf names how a model runs.
+func modeOf(sc catalog.Scaling) string {
+ switch {
+ case sc.Min > 0:
+ return modeAlways
+ case sc.Max > 0:
+ return modeOnDemand
+ default:
+ return modeStopped
+ }
+}
diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go
index 2c12501a..40210bf7 100644
--- a/internal/web/admin_serve_test.go
+++ b/internal/web/admin_serve_test.go
@@ -15,10 +15,12 @@ import (
)
type fakeCatalog struct {
- added []catalog.Model
- declared string
- sources map[string][]string
- err error
+ added []catalog.Model
+ declared string
+ sources map[string][]catalog.Model
+ scaled []string
+ lastScale catalog.Scaling
+ err error
}
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
@@ -29,10 +31,16 @@ func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
return nil
}
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
-func (f *fakeCatalog) Sources(context.Context) (map[string][]string, error) {
+func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, error) {
return f.sources, nil
}
+func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
+ f.scaled = append(f.scaled, name)
+ f.lastScale = sc
+ return catalog.Model{Name: name, Scaling: sc}, f.err
+}
+
// fleet reports a cluster with a given number of cards and engines.
type fakeFleet struct {
gpus int
@@ -405,3 +413,17 @@ func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) {
t.Errorf("the server's answer should be shown; got %s", body)
}
}
+
+func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
+ cat := &fakeCatalog{}
+ mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
+ "csrf": {"tok"}, "name": {"big"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
+ "replicas": {"1"}, "mode": {"ondemand"}, "context_length": {"4096"}, "profile": {"cpu"},
+ })
+ if len(cat.added) != 1 {
+ t.Fatalf("expected one model added, got %d", len(cat.added))
+ }
+ if sc := cat.added[0].Scaling; sc.Min != 0 || sc.Max != 1 || sc.IdleTimeout == 0 {
+ t.Errorf("on demand was added as %+v", sc)
+ }
+}
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index 6559d38e..1781add6 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -445,3 +445,5 @@ h3.limits-for { font-size: .92rem; margin: 1rem 0 .4rem; }
table.narrow-table { max-width: 30rem; }
.q-act a + a { margin-left: .8rem; }
.chip-role { font-size: .68rem; font-weight: 600; color: var(--accent); text-transform: uppercase; letter-spacing: .04em; }
+table.held .use { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap; margin: .1rem 0; }
+table.held select.mode { padding: .1rem .3rem; font-size: .8rem; }
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index 5808d04f..4d825bd3 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -49,7 +49,25 @@ reachable from this machine.
{{if eq (printf "%s" .State) "fetching"}}downloading
{{else if eq (printf "%s" .State) "failed"}}failed
- {{else if .UsedBy}}{{range .UsedBy}}{{.}}{{end}}
+ {{else if .UsedBy}}{{range .UsedBy}}
+ {{.Name}}
+ {{if .Declared}}{{if eq .Mode "always"}}always on{{else if eq .Mode "ondemand"}}on demand{{else}}stopped{{end}} · set in server.yaml
+ {{else}}
+ {{/* Submitted on change: the choice is the action, and a second click
+ to confirm it would only be a way to forget to. */}}
+
+ {{end}}
+
{{end}}
{{else}}not served{{end}}
@@ -100,6 +118,11 @@ reachable from this machine.
+
{{/* Reaches clients through /v1/models, which for most of them is the only
view of what this gateway serves. Nine names and nothing to choose