From e034862b79d50028d3a2e1003c7ce2533b7751d7 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 15:54:20 +0200 Subject: [PATCH] Serve a mirrored model from the list, instead of writing YAML for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetching a model put it on the mirror and left the operator to hand-write a catalogue entry and run `ctl model add` on the control plane — for a source the page was already displaying, with an engine decided by the file extension it had just shown them. A present row now offers to serve it. The form opens on that row with the name, the mirror's own URL and the engine filled in, and asks for the two things nobody can infer: the context pool and how many replicas. What it refuses is the point. A version of several files is refused with the reason. A catalogue entry names one source, so an entry for a sharded model is an entry for one shard, which loads as a broken model rather than failing to load. Nothing said so before — the mirror happily holds a four-shard model that no catalogue entry can describe. Safetensors on a fleet with no GPU is warned about before anything is added. This is the failure we hit today: fifteen gigabytes of shards arrived, and only then did looking at a node show no card, no vLLM and no SGLang. The control plane is not told which engines a node has installed, so the check is on cards, which it does know — a warning rather than a refusal, because a card can be added tomorrow. A name already declared in configuration is refused here too, matching what the model service would say anyway, but before the form is filled in rather than after. Without a store there is no serve action at all: a model added at runtime would not survive a restart, and offering it then refusing is worse than not offering. The mirror page's dependencies became a struct. Three are optional interfaces and two are the same shape, which at a call site is a swap nobody notices until a page shows the wrong thing. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/server.go | 3 + internal/router/handlers.go | 5 +- internal/router/router.go | 6 + internal/web/account.go | 4 + internal/web/admin_mirror.go | 73 ++++++++- internal/web/admin_mirror_test.go | 7 +- internal/web/admin_serve.go | 199 +++++++++++++++++++++++ internal/web/admin_serve_test.go | 198 ++++++++++++++++++++++ internal/web/templates/admin_mirror.html | 48 ++++++ 9 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 internal/web/admin_serve.go create mode 100644 internal/web/admin_serve_test.go diff --git a/internal/cli/server.go b/internal/cli/server.go index 7932557..3adcc0a 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -159,6 +159,9 @@ func newServerCmd() *cobra.Command { admin.ReplaceCatalog(ctx, built) } api.SetModels(svc) + // And the mirror page, so a model that has just been + // fetched can be served without writing YAML by hand. + rt.SetCatalogAdmin(svc) // Benchmark results live here too. A measurement costs real // GPU time; without somewhere to put it the number lives in // whoever ran it's terminal and is gone by morning. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 769ff3b..f1effbf 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -165,7 +165,10 @@ func (r *Router) Handler() http.Handler { // with its own disk, and it is worth administering whether or not this // process also has a cluster to show. if r.mirror != nil { - pages.RegisterAdminMirror(mux, r.mirror, r.registry) + pages.RegisterAdminMirror(mux, web.MirrorDeps{ + Mirror: r.mirror, Registry: r.registry, + Catalog: r.catalogAdmin, Fleet: r.clusterView, + }) } } diff --git a/internal/router/router.go b/internal/router/router.go index 1137a5a..025585a 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -105,6 +105,9 @@ type Router struct { // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry + // catalogAdmin puts a mirrored model into service. Nil without a store, + // where a model added at runtime would not survive a restart. + catalogAdmin web.CatalogAdmin } // SetClusterView gives the router the data the /admin page renders. @@ -125,6 +128,9 @@ func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m } // SetRegistry gives the mirror page somewhere to look models up. func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } +// SetCatalogAdmin lets the mirror page put a model into service. +func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c } + // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } diff --git a/internal/web/account.go b/internal/web/account.go index 2baf01c..7e017cc 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -59,6 +59,10 @@ type AccountPages struct { hasQuotas bool // registry browses a model registry, or is nil where there is no way out. registry Registry + // catalog puts a mirrored model into service; fleet says whether anything + // could run it. Both nil where the mirror page is read-only. + catalog CatalogAdmin + fleet Source } // ServedModels supplies the names the profile page's example should use. diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 835a079..7631d66 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -60,6 +60,11 @@ type mirrorRow struct { // somebody will copy, so it is rendered rather than left to be assembled. URL string Percent int + // Serve is the offer to put this version into service, open on the row the + // operator clicked. Nil on a row that is still arriving: a catalog entry + // pointing at a partial file is the thing versioned paths exist to prevent. + Serve *serveForm + // HasPercent is false when the upstream never sent a length. The bar is // then not drawn at all: a progress bar that has to invent its own total is // worse than a byte count that only goes up. @@ -80,6 +85,10 @@ type adminMirrorData struct { // a mirror waiting to be filled, the other is a machine to go and look at. Unreachable bool + // HasCatalog is false where the catalogue cannot be changed at runtime, and + // the serve action is then not offered rather than offered and refused. + HasCatalog bool + // The explorer. HasRegistry bool Query string @@ -107,9 +116,29 @@ type adminMirrorData struct { // need a shell for, while anything that would let a stolen session reach // further — pointing the control plane at a different mirror, say — stays in // configuration where changing it means access to the host. -func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, reg Registry) { +// MirrorDeps is what the mirror page can reach. +// +// A struct rather than five parameters: three of them are optional interfaces +// and two are the same shape, which at a call site is a swap nobody notices +// until a page shows the wrong thing. +type MirrorDeps struct { + Mirror MirrorAdmin + // Registry browses a model registry. Nil turns the explorer off. + Registry Registry + // Catalog puts a mirrored model into service. Nil leaves the serve action + // off, which is right where the catalogue cannot be changed at runtime. + Catalog CatalogAdmin + // Fleet is the cluster, for saying whether anything could run a model + // before somebody spends an hour downloading it. + Fleet Source +} + +func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { + m := d.Mirror p.hasMirror = true - p.registry = reg + p.registry = d.Registry + p.catalog = d.Catalog + p.fleet = d.Fleet mux.HandleFunc("GET /admin/mirror", p.adminOnly( func(w http.ResponseWriter, r *http.Request, _ store.User) { @@ -153,6 +182,24 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re model, version, plural(started, "file", "files")), nil })) + if d.Catalog != nil { + mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + model, err := buildModel(r.FormValue) + if err != nil { + return "", err + } + if err := d.Catalog.Add(ctx, model, admin.Email); err != nil { + return "", err + } + // Placed on the next pass, not now: saying it is serving would + // be wrong for however long the weights take to reach a node. + return fmt.Sprintf( + "%s added to the catalog, %s; it is placed on the next pass", + model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil + })) + } + mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m, func(ctx context.Context, _ store.User, r *http.Request) (string, error) { model, ver := r.PathValue("model"), r.PathValue("version") @@ -204,6 +251,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, Now: time.Now().UTC().Format(time.RFC3339), } + d.HasCatalog = p.catalog != nil p.explore(r, &d) got, err := m.List(r.Context()) @@ -238,6 +286,27 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, d.Rows = append(d.Rows, row) } + // The one row whose serve form is open, and only when the catalog can be + // changed at all. + if p.catalog != nil { + if want := r.URL.Query().Get("serve"); want != "" { + byVersion := map[string][]mirror.Status{} + for _, st := range got.Models { + byVersion[st.Model+"/"+st.Version] = append(byVersion[st.Model+"/"+st.Version], st) + } + for i := range d.Rows { + key := d.Rows[i].Model + ":" + d.Rows[i].Version + if key != want || d.Rows[i].State != mirror.Present { + continue + } + form := planServe(d.Rows[i].Status, + byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog) + d.Rows[i].Serve = &form + break + } + } + } + // Downloads first, then failures, then what is simply here. The two states // somebody opened this page for are at the top; the rest is a list they // scroll. Within a state the order is fixed so rows do not shuffle between diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index 1327973..664e608 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -93,12 +93,17 @@ func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux { // mirrorPageWith mounts it with a registry behind the explorer. func mirrorPageWith(t *testing.T, m MirrorAdmin, reg Registry, admin bool) *http.ServeMux { + return mirrorPageDeps(t, MirrorDeps{Mirror: m, Registry: reg}, admin) +} + +// mirrorPageDeps mounts it with everything chosen. +func mirrorPageDeps(t *testing.T, d MirrorDeps, admin bool) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}} p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) mux := http.NewServeMux() - p.RegisterAdminMirror(mux, m, reg) + p.RegisterAdminMirror(mux, d) return mux } diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go new file mode 100644 index 0000000..4b871c0 --- /dev/null +++ b/internal/web/admin_serve.go @@ -0,0 +1,199 @@ +package web + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +// CatalogAdmin adds a model to the catalogue in force. +type CatalogAdmin interface { + Add(ctx context.Context, m catalog.Model, by string) error + Declared(name string) bool +} + +// serveForm is the offer to put one mirrored version into service. +// +// Everything a catalogue entry needs that the mirror already knows is filled +// in: the name, the source, and the engine that reads that kind of file. What +// is left is what nobody can infer — the context pool and how many replicas — +// and those are fields rather than guesses presented as facts. +type serveForm struct { + Model string + Version string + Source string + Engine string + // Profile is the profile name, which doubles as how a node_selector would + // be written later. Named for the hardware rather than the model, because + // a profile is about where a model runs. + Profile string + + // Refusal is set when this version cannot be served as it stands, and says + // why. Offered and then failing at placement is worse than not offered. + Refusal string + // Warning is set when it can be served but probably will not run. + Warning string + // Already is true when a model of this name is in the catalogue. + Already bool +} + +// planServe works out what serving this version would mean. +func planServe(st mirror.Status, files []mirror.Status, base string, + fleet Source, cat CatalogAdmin, +) serveForm { + f := serveForm{ + Model: st.Model, + Version: st.Version, + Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}. + URL(base), + Engine: engineFor(st.File), + Profile: "cpu", + } + + // A catalogue entry names one source. A version made of several files has + // no single one, so an entry for it would be an entry for one shard — which + // loads as a broken model rather than failing to load. + if len(files) > 1 { + f.Refusal = fmt.Sprintf( + "this version is %d files, and a catalogue entry names one source. "+ + "Serve a single-file build (a GGUF), or give the engine the "+ + "repository id so it fetches the shards itself", + len(files)) + return f + } + + switch f.Engine { + case "": + f.Refusal = "nothing here reads " + extOf(st.File) + " files" + return f + case "llamacpp": + f.Profile = "cpu" + default: + f.Profile = "gpu" + // The control plane is not told which engines a node has installed, so + // this is the nearest thing it can check honestly. It is also the thing + // that was wrong here: fifteen gigabytes of safetensors arrived before + // anybody noticed the fleet had no card to run them on. + if gpus(fleet) == 0 { + f.Warning = "no node in this fleet reports a GPU, and " + + extOf(st.File) + " weights need vLLM or SGLang, which do. " + + "A GGUF build of the same model runs on the CPU nodes" + } + } + if cat != nil && cat.Declared(st.Model) { + f.Already = true + f.Refusal = "a model called " + st.Model + + " is declared in configuration; edit it there rather than adding a second" + } + return f +} + +// engineFor is the engine that reads this kind of weights file. +func engineFor(file string) string { + switch strings.ToLower(extOf(file)) { + case ".gguf": + return "llamacpp" + case ".safetensors", ".bin": + return "vllm" + } + return "" +} + +func extOf(file string) string { + if i := strings.LastIndex(file, "."); i >= 0 { + return file[i:] + } + return file +} + +func gpus(fleet Source) int { + if fleet == nil { + return 0 + } + n := 0 + for _, node := range fleet.Nodes() { + n += node.GPUTotal + } + return n +} + +// buildModel turns the submitted form into a catalogue entry. +func buildModel(get func(string) string) (catalog.Model, error) { + m := catalog.Model{ + Name: strings.TrimSpace(get("name")), + Source: strings.TrimSpace(get("source")), + RoutingProfile: "chat", + } + if m.Name == "" || m.Source == "" { + return m, errors.New("a model needs a name and a source") + } + for _, a := range strings.Split(get("aliases"), ",") { + if a = strings.TrimSpace(a); a != "" { + m.Aliases = append(m.Aliases, a) + } + } + + ctxLen, err := positive(get("context_length"), "the context length") + if err != nil { + return m, err + } + m.ContextLength = ctxLen + if v := strings.TrimSpace(get("max_output")); v != "" { + if m.MaxOutput, err = positive(v, "the output limit"); err != nil { + return m, err + } + } + + min, err := positive(get("replicas"), "the replica count") + if err != nil { + return m, err + } + m.Scaling = catalog.Scaling{Min: min, Max: min} + + engine := strings.TrimSpace(get("engine")) + if engine == "" { + return m, errors.New("which engine should run it?") + } + p := catalog.Profile{ + Name: strings.TrimSpace(get("profile")), Engine: engine, + Weight: 1, Priority: 100, + } + if p.Name == "" { + p.Name = engine + } + // The pool, not the per-request limit: llama.cpp divides it across its + // parallel slots, so the number a client may use is smaller and is what + // context_length above says. + if v := strings.TrimSpace(get("ctx_size")); v != "" { + if _, err := positive(v, "the engine's context pool"); err != nil { + return m, err + } + p.Args = []string{"--ctx-size", v} + } + if v := strings.TrimSpace(get("min_memory_gb")); v != "" { + if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { + return m, err + } + } + if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { + // What keeps a large model off the small nodes. A CPU profile has no + // card to be short of, so without this every ready node looks like a + // candidate and the first one picked is the one that gets OOM-killed. + p.NodeSelector = map[string]string{k: v} + } + m.Profiles = []catalog.Profile{p} + return m, nil +} + +func positive(v, what string) (int, error) { + n, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil || n < 0 { + return 0, fmt.Errorf("%s must be a whole number", what) + } + return n, nil +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go new file mode 100644 index 0000000..6f74e96 --- /dev/null +++ b/internal/web/admin_serve_test.go @@ -0,0 +1,198 @@ +package web + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +type fakeCatalog struct { + added []catalog.Model + declared string + err error +} + +func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { + if f.err != nil { + return f.err + } + f.added = append(f.added, m) + return nil +} +func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } + +// fleet reports a cluster with a given number of cards. +type fakeFleet struct{ gpus int } + +func (f fakeFleet) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus}} +} +func (f fakeFleet) Models() []adminapi.ModelView { return nil } +func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } +func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } +func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} } + +func held(model, version, file string) mirror.Status { + return mirror.Status{Model: model, Version: version, File: file, + State: mirror.Present, BytesDone: 10, BytesTotal: 10} +} + +func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int) *http.ServeMux { + t.Helper() + return mirrorPageDeps(t, MirrorDeps{ + Mirror: &fakeMirror{models: models}, Catalog: cat, Fleet: fakeFleet{gpus: gpus}, + }, true) +} + +func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) { + cat := &fakeCatalog{} + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0), + "/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String() + + // The mirror already knows the name, the URL and what reads that kind of + // file; leaving an operator to write YAML for it was the gap. + if !strings.Contains(body, `name="engine" value="llamacpp"`) { + t.Errorf("the engine should be inferred from the extension; got %s", body) + } + if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") { + t.Error("the source should be the mirror's own URL") + } +} + +func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { + models := []mirror.Status{ + held("llama", "v1", "model-00001-of-00002.safetensors"), + held("llama", "v1", "model-00002-of-00002.safetensors"), + } + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8), + "/admin/mirror?serve=llama:v1").Body.String() + + // A catalogue entry names one source, so an entry for a sharded version is + // an entry for one shard — which loads as a broken model rather than + // failing to load. + if !strings.Contains(body, "a catalogue entry names one source") { + t.Errorf("the refusal should say why; got %s", body) + } + if strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("no form should be offered for something that cannot be served") + } +} + +func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + // Fifteen gigabytes arrived before anybody noticed the fleet had no card + // to run them on. The page can say so. + if !strings.Contains(body, "no node in this fleet reports a GPU") { + t.Errorf("expected the warning; got %s", body) + } + // A warning, not a refusal: a card could be added tomorrow. + if !strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("it should still be possible to add it") + } +} + +func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) { + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8), + "/admin/mirror?serve=qwen:v1").Body.String() + + if strings.Contains(body, "reports a GPU") { + t.Error("a fleet with cards should not be warned about") + } + if !strings.Contains(body, `value="vllm"`) { + t.Error("safetensors should propose vllm") + } +} + +func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) { + cat := &fakeCatalog{declared: "qwen"} + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "m.gguf")}, cat, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + if !strings.Contains(body, "declared in configuration") { + t.Errorf("expected the refusal; got %s", body) + } +} + +func TestServingBuildsTheEntryFromTheForm(t *testing.T) { + cat := &fakeCatalog{} + mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"}, + "source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"}, + "replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"}, + "ctx_size": {"16384"}, "profile": {"cpu-large"}, + "min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"}, + }) + + if len(cat.added) != 1 { + t.Fatalf("expected one model added, got %d", len(cat.added)) + } + m := cat.added[0] + if m.Name != "coder" || len(m.Aliases) != 2 { + t.Errorf("name and aliases did not arrive: %+v", m) + } + if m.Scaling.Min != 3 || m.Scaling.Max != 3 { + t.Errorf("expected three replicas, got %+v", m.Scaling) + } + if len(m.Profiles) != 1 { + t.Fatalf("expected one profile, got %+v", m.Profiles) + } + p := m.Profiles[0] + if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 { + t.Errorf("the profile is wrong: %+v", p) + } + // The label is what keeps a large model off the small machines. + if p.NodeSelector["role"] != "cpu-large" { + t.Errorf("the node selector was lost: %+v", p.NodeSelector) + } + if strings.Join(p.Args, " ") != "--ctx-size 16384" { + t.Errorf("the engine's context pool was lost: %v", p.Args) + } +} + +func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) { + cat := &fakeCatalog{} + mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"}, + "engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"}, + }) + if len(cat.added) != 0 { + t.Error("nothing should have reached the catalog") + } +} + +func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) { + cat := &fakeCatalog{err: errors.New("that name collides with an alias")} + body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"}, + "engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"}, + }).Body.String() + + if !strings.Contains(body, "collides with an alias") { + t.Errorf("the catalog's reason should be shown; got %s", body) + } +} + +func TestWithNoCatalogThereIsNoServeAction(t *testing.T) { + body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{ + Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}}, + }, true), "/admin/mirror").Body.String() + + // Without a store a model added at runtime would not survive a restart, so + // the action is absent rather than offered and then refused. + if strings.Contains(body, "?serve=") { + t.Error("serving should not be offered without somewhere to persist it") + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index c827653..bb5829d 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -68,6 +68,9 @@ reachable from this machine.

{{else}} + {{if $.HasCatalog}}{{if .Serve}}close + {{else}}serve{{end}} + {{end}}
@@ -78,6 +81,51 @@ reachable from this machine.

{{if .Err}}{{.Err}}{{end}} {{if not .Err}}{{.URL}}{{end}} + +{{with .Serve}} + +{{if .Refusal}} +{{/* Offered and then failing at placement is worse than not offered at all. */}} +

Cannot be served as it stands: {{.Refusal}}.

+{{else}} +{{if .Warning}}

{{.Warning}}.

{{end}} + + + + +

Serving {{.Source}} with {{.Engine}}.

+
+ + + +
+
+ + + +
+
+ + + +
+ +
+

The context pool is what the engine is given; llama.cpp divides +it across its parallel slots, so the number one request may use is the smaller +one above. A node label keeps a large model off the small machines — a CPU +profile has no card to be short of, so without one every ready node looks like +a candidate and the first one picked is the one that runs out of memory.

+{{end}} + +{{end}} {{end}}

The address under each row is what a catalog entry should say.