Set how a model runs from the mirror page
Every mirrored version was in use on the cluster, so the serve link — hidden for a version already served — appeared nowhere, and changing a model between always on and on demand needed the CLI. Each model using a file now shows how it runs and offers always on, on demand (15 minutes idle) or stopped, through the model service's Scale; models declared in server.yaml show their mode and where to change it. The serve form asks the same question. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d8cefd8b21
commit
bf94cd1667
8 files changed
+188
-18
No files matched your search
@@ -29,6 +29,14 @@ organization's usage over time by team, project or person, its teams and
|
||||
projects with limits and spend, and its members — read-only, and only what was
|
||||
spent in its projects. Administrators can open every organization's page.
|
||||
|
||||
### How a model runs, from the mirror page
|
||||
|
||||
Each model a mirrored file is serving now shows how it runs — always on, on
|
||||
demand, or stopped — and it can be changed there, without the CLI. The serve
|
||||
form asks the same: on demand loads the model when somebody asks for it and
|
||||
stops it after 15 minutes unused. A model declared in `server.yaml` shows its
|
||||
mode and says to change it there, because the file is read on every start.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Usage by team, project or organization, and the spend on the quotas page,
|
||||
|
||||
@@ -71,16 +71,16 @@ func (s *Service) Declared(name string) bool {
|
||||
}
|
||||
|
||||
// Sources maps each weights source in the catalog to the models that name it,
|
||||
// so a page listing files can say which of them are in use.
|
||||
func (s *Service) Sources(ctx context.Context) (map[string][]string, error) {
|
||||
// so a page listing files can say which of them are in use and how they run.
|
||||
func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, error) {
|
||||
c, err := s.Load(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]string{}
|
||||
out := map[string][]catalog.Model{}
|
||||
for _, m := range c.Models() {
|
||||
if m.Source != "" {
|
||||
out[m.Source] = append(out[m.Source], m.Name)
|
||||
out[m.Source] = append(out[m.Source], m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -54,6 +55,16 @@ type repoSet struct {
|
||||
|
||||
type repoSource struct{ URL, Path string }
|
||||
|
||||
// modelUse is a catalog model that runs a mirrored version, and how.
|
||||
type modelUse struct {
|
||||
Name string
|
||||
Mode string // always, ondemand or stopped
|
||||
Replicas int
|
||||
// Declared models are read from server.yaml on every start, so a mode
|
||||
// changed here would revert; the page says where to change it instead.
|
||||
Declared bool
|
||||
}
|
||||
|
||||
// mirrorRow is one model version as the page shows it.
|
||||
type mirrorRow struct {
|
||||
mirror.Status
|
||||
@@ -75,9 +86,10 @@ type mirrorRow struct {
|
||||
// row — a fifteen-shard model is one thing to serve or remove, and fifteen
|
||||
// rows of it pushed everything else off the page.
|
||||
Files int
|
||||
// UsedBy names the catalog models whose source is this row's address, so
|
||||
// a file that is in use does not look like one waiting to be served.
|
||||
UsedBy []string
|
||||
// UsedBy is the catalog models whose source is this row's address, and how
|
||||
// each runs, so a file in use does not look like one waiting to be served
|
||||
// and how it runs can be changed where it is seen.
|
||||
UsedBy []modelUse
|
||||
}
|
||||
|
||||
type adminMirrorData struct {
|
||||
@@ -193,6 +205,27 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
}))
|
||||
|
||||
if d.Catalog != nil {
|
||||
// How a model runs, changed where the file it runs is seen.
|
||||
mux.HandleFunc("POST /admin/mirror/mode", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
name, mode := r.FormValue("model"), r.FormValue("mode")
|
||||
if name == "" || (mode != modeAlways && mode != modeOnDemand && mode != modeStopped) {
|
||||
return "", errors.New("choose a model and how it should run")
|
||||
}
|
||||
replicas, _ := strconv.Atoi(r.FormValue("replicas"))
|
||||
sc := scalingFor(mode, replicas, 0)
|
||||
if _, err := d.Catalog.Scale(ctx, name, sc, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch mode {
|
||||
case modeOnDemand:
|
||||
return name + " now loads when asked for and stops after 15 minutes unused", nil
|
||||
case modeStopped:
|
||||
return name + " is stopped; its replicas are being taken down", nil
|
||||
}
|
||||
return name + " now runs all the time", nil
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
model, err := buildModel(r.FormValue)
|
||||
@@ -286,7 +319,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
|
||||
d.Base = got.Base
|
||||
var sources map[string][]string
|
||||
var sources map[string][]catalog.Model
|
||||
if p.catalog != nil {
|
||||
if sources, err = p.catalog.Sources(r.Context()); err != nil {
|
||||
p.log.Warn("could not read which models use the mirror", "err", err)
|
||||
@@ -324,7 +357,12 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
d.Rows = append(d.Rows, row)
|
||||
}
|
||||
for i := range d.Rows {
|
||||
d.Rows[i].UsedBy = sources[d.Rows[i].URL]
|
||||
for _, m := range sources[d.Rows[i].URL] {
|
||||
d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, modelUse{
|
||||
Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
|
||||
Declared: p.catalog.Declared(m.Name),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The one row whose serve form is open, and only when the catalog can be
|
||||
|
||||
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -542,7 +543,8 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) {
|
||||
{Model: "big", Version: "v1", File: "a-00002-of-00002.gguf", State: mirror.Present, BytesDone: 1 << 30},
|
||||
{Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
|
||||
}}
|
||||
cat := &fakeCatalog{sources: map[string][]string{"http://mirror:8081/small/v1/s.gguf": {"small-chat"}}}
|
||||
cat := &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
|
||||
{Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}}
|
||||
body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true),
|
||||
"/admin/mirror").Body.String()
|
||||
|
||||
@@ -567,3 +569,32 @@ func TestAHeldVersionIsOneLineAndSaysWhoUsesIt(t *testing.T) {
|
||||
t.Error("a file nobody uses should be offered for serving")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHowAModelRunsIsChangedWhereItsFileIsShown(t *testing.T) {
|
||||
m := &fakeMirror{models: []mirror.Status{
|
||||
{Model: "small", Version: "v1", File: "s.gguf", State: mirror.Present, BytesDone: 1 << 20},
|
||||
}}
|
||||
cat := &fakeCatalog{declared: "fixed", sources: map[string][]catalog.Model{"http://mirror:8081/small/v1/s.gguf": {
|
||||
{Name: "small-chat", Scaling: catalog.Scaling{Min: 1, Max: 1}},
|
||||
{Name: "fixed", Scaling: catalog.Scaling{Min: 0, Max: 1}},
|
||||
}}}
|
||||
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat}, true)
|
||||
|
||||
body := mirrorGet(t, mux, "/admin/mirror").Body.String()
|
||||
if !strings.Contains(body, `<option value="always" selected>always on</option>`) {
|
||||
t.Error("the running model's mode is not shown as chosen")
|
||||
}
|
||||
// Declared in server.yaml: shown, not offered, since it would revert.
|
||||
if !strings.Contains(body, "on demand · set in server.yaml") {
|
||||
t.Error("a declared model should say where its mode is set")
|
||||
}
|
||||
if strings.Count(body, `name="mode" class="mode"`) != 1 {
|
||||
t.Error("only the runtime model should have a mode selector")
|
||||
}
|
||||
|
||||
mirrorPost(t, mux, "/admin/mirror/mode", url.Values{"csrf": {"tok"},
|
||||
"model": {"small-chat"}, "mode": {"ondemand"}, "replicas": {"1"}})
|
||||
if len(cat.scaled) != 1 || cat.lastScale.Min != 0 || cat.lastScale.Max != 1 || cat.lastScale.IdleTimeout == 0 {
|
||||
t.Errorf("on demand was sent as %+v", cat.lastScale)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
@@ -16,7 +17,10 @@ type CatalogAdmin interface {
|
||||
Add(ctx context.Context, m catalog.Model, by string) error
|
||||
Declared(name string) bool
|
||||
// Sources maps a weights source to the catalog models that use it.
|
||||
Sources(ctx context.Context) (map[string][]string, error)
|
||||
Sources(ctx context.Context) (map[string][]catalog.Model, error)
|
||||
// Scale changes how a model added at runtime runs: always, on demand, or
|
||||
// not at all. One declared in configuration is refused.
|
||||
Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// serveForm is the offer to put one mirrored version into service.
|
||||
@@ -211,7 +215,7 @@ func buildModel(get func(string) string) (catalog.Model, error) {
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
m.Scaling = catalog.Scaling{Min: min, Max: min}
|
||||
m.Scaling = scalingFor(get("mode"), min, 0)
|
||||
|
||||
engine := strings.TrimSpace(get("engine"))
|
||||
if engine == "" {
|
||||
@@ -300,3 +304,45 @@ func sep(existing string) string {
|
||||
}
|
||||
return ", and "
|
||||
}
|
||||
|
||||
// Run modes a page offers for a model, in the words a person would use.
|
||||
const (
|
||||
modeAlways = "always"
|
||||
modeOnDemand = "ondemand"
|
||||
modeStopped = "stopped"
|
||||
)
|
||||
|
||||
// defaultIdle is how long an on-demand model set from a page stays loaded
|
||||
// after its last request.
|
||||
const defaultIdle = 15 * time.Minute
|
||||
|
||||
// scalingFor turns a run mode into scaling: replicas is how many run, or may
|
||||
// run on demand; idle keeps an on-demand model's timeout when it has one.
|
||||
func scalingFor(mode string, replicas int, idle time.Duration) catalog.Scaling {
|
||||
if replicas < 1 {
|
||||
replicas = 1
|
||||
}
|
||||
switch mode {
|
||||
case modeOnDemand:
|
||||
if idle <= 0 {
|
||||
idle = defaultIdle
|
||||
}
|
||||
return catalog.Scaling{Min: 0, Max: replicas, IdleTimeout: idle}
|
||||
case modeStopped:
|
||||
return catalog.Scaling{Min: 0, Max: 0}
|
||||
default:
|
||||
return catalog.Scaling{Min: replicas, Max: replicas}
|
||||
}
|
||||
}
|
||||
|
||||
// modeOf names how a model runs.
|
||||
func modeOf(sc catalog.Scaling) string {
|
||||
switch {
|
||||
case sc.Min > 0:
|
||||
return modeAlways
|
||||
case sc.Max > 0:
|
||||
return modeOnDemand
|
||||
default:
|
||||
return modeStopped
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,12 @@ import (
|
||||
)
|
||||
|
||||
type fakeCatalog struct {
|
||||
added []catalog.Model
|
||||
declared string
|
||||
sources map[string][]string
|
||||
err error
|
||||
added []catalog.Model
|
||||
declared string
|
||||
sources map[string][]catalog.Model
|
||||
scaled []string
|
||||
lastScale catalog.Scaling
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||
@@ -29,10 +31,16 @@ func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeCatalog) Declared(name string) bool { return name == f.declared }
|
||||
func (f *fakeCatalog) Sources(context.Context) (map[string][]string, error) {
|
||||
func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, error) {
|
||||
return f.sources, nil
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Scale(_ context.Context, name string, sc catalog.Scaling, _ string) (catalog.Model, error) {
|
||||
f.scaled = append(f.scaled, name)
|
||||
f.lastScale = sc
|
||||
return catalog.Model{Name: name, Scaling: sc}, f.err
|
||||
}
|
||||
|
||||
// fleet reports a cluster with a given number of cards and engines.
|
||||
type fakeFleet struct {
|
||||
gpus int
|
||||
@@ -405,3 +413,17 @@ func TestASourceTheMirrorNoLongerHoldsIsRefused(t *testing.T) {
|
||||
t.Errorf("the server's answer should be shown; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServingOnDemandLoadsWhenAskedFor(t *testing.T) {
|
||||
cat := &fakeCatalog{}
|
||||
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
||||
"csrf": {"tok"}, "name": {"big"}, "source": {reachableSource(t)}, "engine": {"llamacpp"},
|
||||
"replicas": {"1"}, "mode": {"ondemand"}, "context_length": {"4096"}, "profile": {"cpu"},
|
||||
})
|
||||
if len(cat.added) != 1 {
|
||||
t.Fatalf("expected one model added, got %d", len(cat.added))
|
||||
}
|
||||
if sc := cat.added[0].Scaling; sc.Min != 0 || sc.Max != 1 || sc.IdleTimeout == 0 {
|
||||
t.Errorf("on demand was added as %+v", sc)
|
||||
}
|
||||
}
|
||||
@@ -445,3 +445,5 @@ h3.limits-for { font-size: .92rem; margin: 1rem 0 .4rem; }
|
||||
table.narrow-table { max-width: 30rem; }
|
||||
.q-act a + a { margin-left: .8rem; }
|
||||
.chip-role { font-size: .68rem; font-weight: 600; color: var(--accent); text-transform: uppercase; letter-spacing: .04em; }
|
||||
table.held .use { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap; margin: .1rem 0; }
|
||||
table.held select.mode { padding: .1rem .3rem; font-size: .8rem; }
|
||||
@@ -49,7 +49,25 @@ reachable from this machine.</p>
|
||||
<td>
|
||||
{{if eq (printf "%s" .State) "fetching"}}<span class="state state-starting">downloading</span>
|
||||
{{else if eq (printf "%s" .State) "failed"}}<span class="state state-failed">failed</span>
|
||||
{{else if .UsedBy}}{{range .UsedBy}}<span class="tag">{{.}}</span>{{end}}
|
||||
{{else if .UsedBy}}{{range .UsedBy}}<div class="use">
|
||||
<span class="tag">{{.Name}}</span>
|
||||
{{if .Declared}}<span class="opt">{{if eq .Mode "always"}}always on{{else if eq .Mode "ondemand"}}on demand{{else}}stopped{{end}} · set in server.yaml</span>
|
||||
{{else}}
|
||||
{{/* Submitted on change: the choice is the action, and a second click
|
||||
to confirm it would only be a way to forget to. */}}
|
||||
<form method="post" action="/admin/mirror/mode" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="model" value="{{.Name}}">
|
||||
<input type="hidden" name="replicas" value="{{.Replicas}}">
|
||||
<select name="mode" class="mode" onchange="this.form.submit()" aria-label="How {{.Name}} runs">
|
||||
<option value="always"{{if eq .Mode "always"}} selected{{end}}>always on</option>
|
||||
<option value="ondemand"{{if eq .Mode "ondemand"}} selected{{end}}>on demand</option>
|
||||
<option value="stopped"{{if eq .Mode "stopped"}} selected{{end}}>stopped</option>
|
||||
</select>
|
||||
<noscript><button type="submit" class="link">set</button></noscript>
|
||||
</form>
|
||||
{{end}}
|
||||
</div>{{end}}
|
||||
{{else}}<span class="opt">not served</span>{{end}}
|
||||
</td>
|
||||
<td class="actions">
|
||||
@@ -100,6 +118,11 @@ reachable from this machine.</p>
|
||||
<label>Aliases <span class="opt">comma separated</span>
|
||||
<input type="text" name="aliases" placeholder="coder-7b"></label>
|
||||
<label>Replicas <input type="text" name="replicas" value="1" required></label>
|
||||
<label>Runs
|
||||
<select name="mode">
|
||||
<option value="always">always on</option>
|
||||
<option value="ondemand">on demand: loads when asked, stops after 15 min unused</option>
|
||||
</select></label>
|
||||
</div>
|
||||
{{/* Reaches clients through /v1/models, which for most of them is the only
|
||||
view of what this gateway serves. Nine names and nothing to choose
|
||||
|
||||
Reference in new issue
Block a user