diff --git a/internal/cli/server.go b/internal/cli/server.go index 7932557c..3adcc0a9 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -159,6 +159,9 @@ func newServerCmd() *cobra.Command { admin.ReplaceCatalog(ctx, built) } api.SetModels(svc) + // And the mirror page, so a model that has just been + // fetched can be served without writing YAML by hand. + rt.SetCatalogAdmin(svc) // Benchmark results live here too. A measurement costs real // GPU time; without somewhere to put it the number lives in // whoever ran it's terminal and is gone by morning. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 769ff3bd..f1effbfc 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -165,7 +165,10 @@ func (r *Router) Handler() http.Handler { // with its own disk, and it is worth administering whether or not this // process also has a cluster to show. if r.mirror != nil { - pages.RegisterAdminMirror(mux, r.mirror, r.registry) + pages.RegisterAdminMirror(mux, web.MirrorDeps{ + Mirror: r.mirror, Registry: r.registry, + Catalog: r.catalogAdmin, Fleet: r.clusterView, + }) } } diff --git a/internal/router/router.go b/internal/router/router.go index 1137a5a6..025585a1 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -105,6 +105,9 @@ type Router struct { // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry + // catalogAdmin puts a mirrored model into service. Nil without a store, + // where a model added at runtime would not survive a restart. + catalogAdmin web.CatalogAdmin } // SetClusterView gives the router the data the /admin page renders. @@ -125,6 +128,9 @@ func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m } // SetRegistry gives the mirror page somewhere to look models up. func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } +// SetCatalogAdmin lets the mirror page put a model into service. +func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c } + // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } diff --git a/internal/web/account.go b/internal/web/account.go index 2baf01c5..7e017cc3 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -59,6 +59,10 @@ type AccountPages struct { hasQuotas bool // registry browses a model registry, or is nil where there is no way out. registry Registry + // catalog puts a mirrored model into service; fleet says whether anything + // could run it. Both nil where the mirror page is read-only. + catalog CatalogAdmin + fleet Source } // ServedModels supplies the names the profile page's example should use. diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 835a0792..7631d668 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -60,6 +60,11 @@ type mirrorRow struct { // somebody will copy, so it is rendered rather than left to be assembled. URL string Percent int + // Serve is the offer to put this version into service, open on the row the + // operator clicked. Nil on a row that is still arriving: a catalog entry + // pointing at a partial file is the thing versioned paths exist to prevent. + Serve *serveForm + // HasPercent is false when the upstream never sent a length. The bar is // then not drawn at all: a progress bar that has to invent its own total is // worse than a byte count that only goes up. @@ -80,6 +85,10 @@ type adminMirrorData struct { // a mirror waiting to be filled, the other is a machine to go and look at. Unreachable bool + // HasCatalog is false where the catalogue cannot be changed at runtime, and + // the serve action is then not offered rather than offered and refused. + HasCatalog bool + // The explorer. HasRegistry bool Query string @@ -107,9 +116,29 @@ type adminMirrorData struct { // need a shell for, while anything that would let a stolen session reach // further — pointing the control plane at a different mirror, say — stays in // configuration where changing it means access to the host. -func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, reg Registry) { +// MirrorDeps is what the mirror page can reach. +// +// A struct rather than five parameters: three of them are optional interfaces +// and two are the same shape, which at a call site is a swap nobody notices +// until a page shows the wrong thing. +type MirrorDeps struct { + Mirror MirrorAdmin + // Registry browses a model registry. Nil turns the explorer off. + Registry Registry + // Catalog puts a mirrored model into service. Nil leaves the serve action + // off, which is right where the catalogue cannot be changed at runtime. + Catalog CatalogAdmin + // Fleet is the cluster, for saying whether anything could run a model + // before somebody spends an hour downloading it. + Fleet Source +} + +func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { + m := d.Mirror p.hasMirror = true - p.registry = reg + p.registry = d.Registry + p.catalog = d.Catalog + p.fleet = d.Fleet mux.HandleFunc("GET /admin/mirror", p.adminOnly( func(w http.ResponseWriter, r *http.Request, _ store.User) { @@ -153,6 +182,24 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re model, version, plural(started, "file", "files")), nil })) + if d.Catalog != nil { + mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, + func(ctx context.Context, admin store.User, r *http.Request) (string, error) { + model, err := buildModel(r.FormValue) + if err != nil { + return "", err + } + if err := d.Catalog.Add(ctx, model, admin.Email); err != nil { + return "", err + } + // Placed on the next pass, not now: saying it is serving would + // be wrong for however long the weights take to reach a node. + return fmt.Sprintf( + "%s added to the catalog, %s; it is placed on the next pass", + model.Name, plural(model.Scaling.Min, "replica", "replicas")), nil + })) + } + mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m, func(ctx context.Context, _ store.User, r *http.Request) (string, error) { model, ver := r.PathValue("model"), r.PathValue("version") @@ -204,6 +251,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, Now: time.Now().UTC().Format(time.RFC3339), } + d.HasCatalog = p.catalog != nil p.explore(r, &d) got, err := m.List(r.Context()) @@ -238,6 +286,27 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, d.Rows = append(d.Rows, row) } + // The one row whose serve form is open, and only when the catalog can be + // changed at all. + if p.catalog != nil { + if want := r.URL.Query().Get("serve"); want != "" { + byVersion := map[string][]mirror.Status{} + for _, st := range got.Models { + byVersion[st.Model+"/"+st.Version] = append(byVersion[st.Model+"/"+st.Version], st) + } + for i := range d.Rows { + key := d.Rows[i].Model + ":" + d.Rows[i].Version + if key != want || d.Rows[i].State != mirror.Present { + continue + } + form := planServe(d.Rows[i].Status, + byVersion[d.Rows[i].Model+"/"+d.Rows[i].Version], got.Base, p.fleet, p.catalog) + d.Rows[i].Serve = &form + break + } + } + } + // Downloads first, then failures, then what is simply here. The two states // somebody opened this page for are at the top; the rest is a list they // scroll. Within a state the order is fixed so rows do not shuffle between diff --git a/internal/web/admin_mirror_test.go b/internal/web/admin_mirror_test.go index 1327973a..664e6083 100644 --- a/internal/web/admin_mirror_test.go +++ b/internal/web/admin_mirror_test.go @@ -93,12 +93,17 @@ func mirrorPage(t *testing.T, m MirrorAdmin, admin bool) *http.ServeMux { // mirrorPageWith mounts it with a registry behind the explorer. func mirrorPageWith(t *testing.T, m MirrorAdmin, reg Registry, admin bool) *http.ServeMux { + return mirrorPageDeps(t, MirrorDeps{Mirror: m, Registry: reg}, admin) +} + +// mirrorPageDeps mounts it with everything chosen. +func mirrorPageDeps(t *testing.T, d MirrorDeps, admin bool) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: admin}} p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) mux := http.NewServeMux() - p.RegisterAdminMirror(mux, m, reg) + p.RegisterAdminMirror(mux, d) return mux } diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go new file mode 100644 index 00000000..4b871c07 --- /dev/null +++ b/internal/web/admin_serve.go @@ -0,0 +1,199 @@ +package web + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +// CatalogAdmin adds a model to the catalogue in force. +type CatalogAdmin interface { + Add(ctx context.Context, m catalog.Model, by string) error + Declared(name string) bool +} + +// serveForm is the offer to put one mirrored version into service. +// +// Everything a catalogue entry needs that the mirror already knows is filled +// in: the name, the source, and the engine that reads that kind of file. What +// is left is what nobody can infer — the context pool and how many replicas — +// and those are fields rather than guesses presented as facts. +type serveForm struct { + Model string + Version string + Source string + Engine string + // Profile is the profile name, which doubles as how a node_selector would + // be written later. Named for the hardware rather than the model, because + // a profile is about where a model runs. + Profile string + + // Refusal is set when this version cannot be served as it stands, and says + // why. Offered and then failing at placement is worse than not offered. + Refusal string + // Warning is set when it can be served but probably will not run. + Warning string + // Already is true when a model of this name is in the catalogue. + Already bool +} + +// planServe works out what serving this version would mean. +func planServe(st mirror.Status, files []mirror.Status, base string, + fleet Source, cat CatalogAdmin, +) serveForm { + f := serveForm{ + Model: st.Model, + Version: st.Version, + Source: mirror.Entry{Model: st.Model, Version: st.Version, File: st.File}. + URL(base), + Engine: engineFor(st.File), + Profile: "cpu", + } + + // A catalogue entry names one source. A version made of several files has + // no single one, so an entry for it would be an entry for one shard — which + // loads as a broken model rather than failing to load. + if len(files) > 1 { + f.Refusal = fmt.Sprintf( + "this version is %d files, and a catalogue entry names one source. "+ + "Serve a single-file build (a GGUF), or give the engine the "+ + "repository id so it fetches the shards itself", + len(files)) + return f + } + + switch f.Engine { + case "": + f.Refusal = "nothing here reads " + extOf(st.File) + " files" + return f + case "llamacpp": + f.Profile = "cpu" + default: + f.Profile = "gpu" + // The control plane is not told which engines a node has installed, so + // this is the nearest thing it can check honestly. It is also the thing + // that was wrong here: fifteen gigabytes of safetensors arrived before + // anybody noticed the fleet had no card to run them on. + if gpus(fleet) == 0 { + f.Warning = "no node in this fleet reports a GPU, and " + + extOf(st.File) + " weights need vLLM or SGLang, which do. " + + "A GGUF build of the same model runs on the CPU nodes" + } + } + if cat != nil && cat.Declared(st.Model) { + f.Already = true + f.Refusal = "a model called " + st.Model + + " is declared in configuration; edit it there rather than adding a second" + } + return f +} + +// engineFor is the engine that reads this kind of weights file. +func engineFor(file string) string { + switch strings.ToLower(extOf(file)) { + case ".gguf": + return "llamacpp" + case ".safetensors", ".bin": + return "vllm" + } + return "" +} + +func extOf(file string) string { + if i := strings.LastIndex(file, "."); i >= 0 { + return file[i:] + } + return file +} + +func gpus(fleet Source) int { + if fleet == nil { + return 0 + } + n := 0 + for _, node := range fleet.Nodes() { + n += node.GPUTotal + } + return n +} + +// buildModel turns the submitted form into a catalogue entry. +func buildModel(get func(string) string) (catalog.Model, error) { + m := catalog.Model{ + Name: strings.TrimSpace(get("name")), + Source: strings.TrimSpace(get("source")), + RoutingProfile: "chat", + } + if m.Name == "" || m.Source == "" { + return m, errors.New("a model needs a name and a source") + } + for _, a := range strings.Split(get("aliases"), ",") { + if a = strings.TrimSpace(a); a != "" { + m.Aliases = append(m.Aliases, a) + } + } + + ctxLen, err := positive(get("context_length"), "the context length") + if err != nil { + return m, err + } + m.ContextLength = ctxLen + if v := strings.TrimSpace(get("max_output")); v != "" { + if m.MaxOutput, err = positive(v, "the output limit"); err != nil { + return m, err + } + } + + min, err := positive(get("replicas"), "the replica count") + if err != nil { + return m, err + } + m.Scaling = catalog.Scaling{Min: min, Max: min} + + engine := strings.TrimSpace(get("engine")) + if engine == "" { + return m, errors.New("which engine should run it?") + } + p := catalog.Profile{ + Name: strings.TrimSpace(get("profile")), Engine: engine, + Weight: 1, Priority: 100, + } + if p.Name == "" { + p.Name = engine + } + // The pool, not the per-request limit: llama.cpp divides it across its + // parallel slots, so the number a client may use is smaller and is what + // context_length above says. + if v := strings.TrimSpace(get("ctx_size")); v != "" { + if _, err := positive(v, "the engine's context pool"); err != nil { + return m, err + } + p.Args = []string{"--ctx-size", v} + } + if v := strings.TrimSpace(get("min_memory_gb")); v != "" { + if p.MinMemoryGB, err = positive(v, "the memory requirement"); err != nil { + return m, err + } + } + if k, v := strings.TrimSpace(get("selector_key")), strings.TrimSpace(get("selector_value")); k != "" && v != "" { + // What keeps a large model off the small nodes. A CPU profile has no + // card to be short of, so without this every ready node looks like a + // candidate and the first one picked is the one that gets OOM-killed. + p.NodeSelector = map[string]string{k: v} + } + m.Profiles = []catalog.Profile{p} + return m, nil +} + +func positive(v, what string) (int, error) { + n, err := strconv.Atoi(strings.TrimSpace(v)) + if err != nil || n < 0 { + return 0, fmt.Errorf("%s must be a whole number", what) + } + return n, nil +} diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go new file mode 100644 index 00000000..6f74e96a --- /dev/null +++ b/internal/web/admin_serve_test.go @@ -0,0 +1,198 @@ +package web + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +type fakeCatalog struct { + added []catalog.Model + declared string + err error +} + +func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { + if f.err != nil { + return f.err + } + f.added = append(f.added, m) + return nil +} +func (f *fakeCatalog) Declared(name string) bool { return name == f.declared } + +// fleet reports a cluster with a given number of cards. +type fakeFleet struct{ gpus int } + +func (f fakeFleet) Nodes() []adminapi.NodeView { + return []adminapi.NodeView{{ID: "n1", GPUTotal: f.gpus}} +} +func (f fakeFleet) Models() []adminapi.ModelView { return nil } +func (f fakeFleet) Replicas(string) []adminapi.ReplicaView { return nil } +func (f fakeFleet) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } +func (f fakeFleet) Summary() adminapi.Summary { return adminapi.Summary{} } + +func held(model, version, file string) mirror.Status { + return mirror.Status{Model: model, Version: version, File: file, + State: mirror.Present, BytesDone: 10, BytesTotal: 10} +} + +func servePage(t *testing.T, models []mirror.Status, cat CatalogAdmin, gpus int) *http.ServeMux { + t.Helper() + return mirrorPageDeps(t, MirrorDeps{ + Mirror: &fakeMirror{models: models}, Catalog: cat, Fleet: fakeFleet{gpus: gpus}, + }, true) +} + +func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) { + cat := &fakeCatalog{} + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen2.5-coder-7b", "c03e6d35", "qwen-q4_k_m.gguf")}, cat, 0), + "/admin/mirror?serve=qwen2.5-coder-7b:c03e6d35").Body.String() + + // The mirror already knows the name, the URL and what reads that kind of + // file; leaving an operator to write YAML for it was the gap. + if !strings.Contains(body, `name="engine" value="llamacpp"`) { + t.Errorf("the engine should be inferred from the extension; got %s", body) + } + if !strings.Contains(body, "/qwen2.5-coder-7b/c03e6d35/qwen-q4_k_m.gguf") { + t.Error("the source should be the mirror's own URL") + } +} + +func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { + models := []mirror.Status{ + held("llama", "v1", "model-00001-of-00002.safetensors"), + held("llama", "v1", "model-00002-of-00002.safetensors"), + } + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8), + "/admin/mirror?serve=llama:v1").Body.String() + + // A catalogue entry names one source, so an entry for a sharded version is + // an entry for one shard — which loads as a broken model rather than + // failing to load. + if !strings.Contains(body, "a catalogue entry names one source") { + t.Errorf("the refusal should say why; got %s", body) + } + if strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("no form should be offered for something that cannot be served") + } +} + +func TestSafetensorsOnACPUFleetWarnsBeforeAnybodyTries(t *testing.T) { + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + // Fifteen gigabytes arrived before anybody noticed the fleet had no card + // to run them on. The page can say so. + if !strings.Contains(body, "no node in this fleet reports a GPU") { + t.Errorf("expected the warning; got %s", body) + } + // A warning, not a refusal: a card could be added tomorrow. + if !strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("it should still be possible to add it") + } +} + +func TestSafetensorsOnAGPUFleetIsNotWarnedAbout(t *testing.T) { + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "model.safetensors")}, &fakeCatalog{}, 8), + "/admin/mirror?serve=qwen:v1").Body.String() + + if strings.Contains(body, "reports a GPU") { + t.Error("a fleet with cards should not be warned about") + } + if !strings.Contains(body, `value="vllm"`) { + t.Error("safetensors should propose vllm") + } +} + +func TestAModelDeclaredInConfigurationIsNotOfferedTwice(t *testing.T) { + cat := &fakeCatalog{declared: "qwen"} + body := mirrorGet(t, servePage(t, []mirror.Status{ + held("qwen", "v1", "m.gguf")}, cat, 0), + "/admin/mirror?serve=qwen:v1").Body.String() + + if !strings.Contains(body, "declared in configuration") { + t.Errorf("expected the refusal; got %s", body) + } +} + +func TestServingBuildsTheEntryFromTheForm(t *testing.T) { + cat := &fakeCatalog{} + mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "aliases": {"c7b, coder-7b"}, + "source": {"http://mirror:8081/coder/v1/m.gguf"}, "engine": {"llamacpp"}, + "replicas": {"3"}, "context_length": {"4096"}, "max_output": {"1024"}, + "ctx_size": {"16384"}, "profile": {"cpu-large"}, + "min_memory_gb": {"8"}, "selector_key": {"role"}, "selector_value": {"cpu-large"}, + }) + + if len(cat.added) != 1 { + t.Fatalf("expected one model added, got %d", len(cat.added)) + } + m := cat.added[0] + if m.Name != "coder" || len(m.Aliases) != 2 { + t.Errorf("name and aliases did not arrive: %+v", m) + } + if m.Scaling.Min != 3 || m.Scaling.Max != 3 { + t.Errorf("expected three replicas, got %+v", m.Scaling) + } + if len(m.Profiles) != 1 { + t.Fatalf("expected one profile, got %+v", m.Profiles) + } + p := m.Profiles[0] + if p.Engine != "llamacpp" || p.Name != "cpu-large" || p.MinMemoryGB != 8 { + t.Errorf("the profile is wrong: %+v", p) + } + // The label is what keeps a large model off the small machines. + if p.NodeSelector["role"] != "cpu-large" { + t.Errorf("the node selector was lost: %+v", p.NodeSelector) + } + if strings.Join(p.Args, " ") != "--ctx-size 16384" { + t.Errorf("the engine's context pool was lost: %v", p.Args) + } +} + +func TestRubbishInTheFormIsRefusedBeforeTheCatalog(t *testing.T) { + cat := &fakeCatalog{} + mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"}, + "engine": {"llamacpp"}, "replicas": {"lots"}, "context_length": {"4096"}, + }) + if len(cat.added) != 0 { + t.Error("nothing should have reached the catalog") + } +} + +func TestTheCatalogsOwnRefusalReachesTheOperator(t *testing.T) { + cat := &fakeCatalog{err: errors.New("that name collides with an alias")} + body := mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{ + "csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"}, + "engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"}, + }).Body.String() + + if !strings.Contains(body, "collides with an alias") { + t.Errorf("the catalog's reason should be shown; got %s", body) + } +} + +func TestWithNoCatalogThereIsNoServeAction(t *testing.T) { + body := mirrorGet(t, mirrorPageDeps(t, MirrorDeps{ + Mirror: &fakeMirror{models: []mirror.Status{held("qwen", "v1", "m.gguf")}}, + }, true), "/admin/mirror").Body.String() + + // Without a store a model added at runtime would not survive a restart, so + // the action is absent rather than offered and then refused. + if strings.Contains(body, "?serve=") { + t.Error("serving should not be offered without somewhere to persist it") + } +} diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index c8276536..bb5829de 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -68,6 +68,9 @@ reachable from this machine.
{{else}} + {{if $.HasCatalog}}{{if .Serve}}close + {{else}}serve{{end}} + {{end}}