From 2bf43ed90c63b3558151436d60dcc9f4b960ff48 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 23:44:51 +0200 Subject: [PATCH] Search Hugging Face from the console and serve what is found Serve a model is a view of the registry's GGUF models, trending or searched; Enter on one lists its files, and the one chosen is served with llmgw ctl model serve. A form of one field is applied by Enter on it. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +- internal/cli/tui_actions.go | 27 +---- internal/cli/tui_actions_test.go | 62 ++++++++++ internal/cli/tui_console.go | 42 ++++++- internal/cli/tui_hf.go | 193 +++++++++++++++++++++++++++++++ 5 files changed, 300 insertions(+), 31 deletions(-) create mode 100644 internal/cli/tui_hf.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 759d9cd..2444ca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,11 @@ what can be done to it, as forms and questions: issue (shown once), limits, revoke; organizations, teams and projects — add, rename, move, remove, limits for the level and for each member, capacity quotas, how long conversations and agent notes are kept. -- **Do**: serve a model from Hugging Face, upgrade the cluster, restart every - replica, back up now. +- **Serve a model** (new view): GGUF models on Hugging Face — trending, most + downloaded, most liked or newest, or searched with / — then one of the + chosen repository's files, a name and a replica count, and it is fetched, + checked, added and started. The registry is mirror.registry's when set. +- **Do**: upgrade the cluster, restart every replica, back up now. The people side runs the same commands as `llmgw users`, `keys` and `org`, against the same store, so their checks and messages are the ones shown. A diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index e9a614f..ae67aef 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -144,7 +144,8 @@ func (m consoleModel) actionKey(k tea.KeyMsg) (tea.Model, tea.Cmd, bool) { fl.opt = (fl.opt + d) % len(fl.options) } case tea.KeyEnter: - if f.focus < len(f.fields) { + // A form of one field is applied by Enter on it. + if f.focus < len(f.fields) && len(f.fields) > 1 { f.fields[f.focus].text.Blur() f.focus++ if f.focus < len(f.fields) { @@ -635,30 +636,6 @@ func (m *consoleModel) runAfter(args ...string) tea.Cmd { func (m *consoleModel) doEntry() tea.Cmd { server := "--server=" + m.client.base switch m.menu { - case cServe: - m.openForm(&formDialog{title: "Serve a model", - note: "A GGUF repository on Hugging Face, like bartowski/Qwen2.5-3B-Instruct-GGUF.", - fields: []formField{ - textF("repository", "", "OWNER/REPO"), - textF("quantisation", "Q4_K_M", ""), - textF("name", "", "in the catalog; empty takes it from the repository"), - textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "4096", "what one request may use"), - }, - submit: func(m *consoleModel, v []string) (tea.Cmd, error) { - if !strings.Contains(v[0], "/") { - return nil, fmt.Errorf("the repository is OWNER/REPO") - } - args := []string{"ctl", server, "model", "serve", v[0], "--quant=" + v[1], "--replicas=" + v[3], - "--context-length=" + v[4]} - if v[3] == "0" { - args = append(args, "--on-demand") - } - if v[2] != "" { - args = append(args, "--name="+v[2]) - } - return m.runAfter(args...), nil - }}) case cUpgrade: m.openForm(&formDialog{title: "Upgrade the cluster", note: "This machine runs " + version.Version + ".", diff --git a/internal/cli/tui_actions_test.go b/internal/cli/tui_actions_test.go index 79e05c8..1e5da5a 100644 --- a/internal/cli/tui_actions_test.go +++ b/internal/cli/tui_actions_test.go @@ -257,3 +257,65 @@ func TestTheSidebarFitsAShortScreen(t *testing.T) { } } } + +func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { + reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/models": + if r.URL.Query().Get("search") != "qwen" || r.URL.Query().Get("filter") != "gguf" { + _, _ = w.Write([]byte(`[]`)) + return + } + _, _ = w.Write([]byte(`[{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","downloads":1200,"gguf":{"total":3000000000,"architecture":"qwen2"}}]`)) + case r.URL.Path == "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF": + _, _ = w.Write([]byte(`{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","sha":"abc"}`)) + case strings.HasPrefix(r.URL.Path, "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF/tree/"): + _, _ = w.Write([]byte(`[{"type":"file","path":"Qwen2.5-3B-Instruct-Q4_K_M.gguf","lfs":{"oid":"x","size":2000000000}}, + {"type":"file","path":"Qwen2.5-3B-Instruct-Q8_0.gguf","lfs":{"oid":"y","size":3000000000}}, + {"type":"file","path":"mmproj-f16.gguf","size":10}]`)) + default: + http.NotFound(w, r) + } + })) + defer reg.Close() + dir := t.TempDir() + cfgPath := dir + "/server.yaml" + if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil { + t.Fatal(err) + } + m := newConsoleModel("127.0.0.1:9090", cfgPath) + m.loaded, m.menu, m.inPanel = true, cServe, true + var tm tea.Model = m + + run := func(tm tea.Model, cmd tea.Cmd) tea.Model { + if cmd != nil { + tm, _ = tm.Update(cmd()) + } + return tm + } + tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("/")}) + tm = press(tm, "q", "w", "e", "n") + tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) + tm = run(tm, cmd) + if v := tm.View(); !strings.Contains(v, "bartowski/Qwen2.5-3B-Instruct-GGUF") || !strings.Contains(v, "3.0B") { + t.Fatalf("no result:\n%s", v) + } + tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) + tm = run(tm, cmd) + d := tm.(consoleModel).menuD + if d == nil || len(d.items) != 2 { + t.Fatalf("files offered: %+v", d) + } + tm = press(tm, "down", "enter") + f := tm.(consoleModel).formD + if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" { + t.Fatalf("form: %+v", f) + } + f.focus = len(f.fields) + tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) + act := tm.(consoleModel).action + got := strings.Join(act.targets, " ") + if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") { + t.Fatalf("would run %q %q", act.kind, got) + } +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index b93c0e2..c48022a 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -20,6 +20,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" ) @@ -151,6 +152,7 @@ type consoleModel struct { st *storeHolder people peopleData mirror consoleMirrorMsg + hf hfState } var usageGroupings = []string{"team", "project", "org"} @@ -286,6 +288,21 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.logs = msg case consoleMirrorMsg: m.mirror = msg + case consoleHFMsg: + if msg.query == m.hf.query && msg.sort == m.hf.sort { + m.hf.models, m.hf.err = msg.models, msg.err + if m.hf.models == nil && msg.err == nil { + m.hf.models = []hf.Model{} + } + } + case consoleRepoMsg: + m.busy = "" + if msg.err != nil { + m.dialog, m.dialogTitle = "error", "Hugging Face" + m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key") + } else { + m.menuD = m.repoMenu(msg.repo) + } case consolePeopleMsg: m.people = peopleData(msg) case consoleQuotasMsg: @@ -366,7 +383,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { return m.ask("Replicate the store", "user@address of database machine", 2) case cLeave: return m.leave() - case cServe, cUpgrade, cRestartAll, cBackup: + case cUpgrade, cRestartAll, cBackup: return m, m.doEntry() case cOverview: return m, nil @@ -408,6 +425,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.menuD = m.quotaMenu(m.quotas[m.row]) case cMirror: m.menuD = m.mirrorMenu(m.mirror.res.Models[m.row]) + case cServe: + return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: m.menuD = m.userMenu(m.people.users[m.row]) case cKeys: @@ -415,6 +434,16 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cOrgs: m.menuD = m.orgMenu(m.people.orgs[m.row]) } + case "/": + if m.menu == cServe { + m.hfSearchDialog() + } + case "s": + if m.menu == cServe { + m.hf.sort = (m.hf.sort + 1) % len(hfSorts) + m.hf.models, m.hf.err, m.row = nil, nil, 0 + return m, m.fetchHF() + } case "a": switch m.menu { case cQuotas: @@ -456,6 +485,10 @@ func (m consoleModel) onEnter() tea.Cmd { return m.fetchQuotas() case cMirror: return m.fetchMirror() + case cServe: + if m.hf.models == nil { + return m.fetchHF() + } case cAccounts, cKeys, cOrgs: // The quotas too, for an organization's quota form. return tea.Batch(m.fetchPeople(), m.fetchQuotas()) @@ -766,6 +799,8 @@ func (m consoleModel) rowCount() int { return len(m.quotas) case cMirror: return len(m.mirror.res.Models) + case cServe: + return len(m.hf.models) case cAccounts: return len(m.people.users) case cKeys: @@ -966,6 +1001,8 @@ func (m consoleModel) panel() (title, body, keys string) { return m.peoplePanel(pw) case cMirror: return m.mirrorPanel(pw) + case cServe: + return m.hfPanel(pw) case cSettings: return m.settingsPanel(pw) } @@ -978,9 +1015,6 @@ func (m consoleModel) panel() (title, body, keys string) { item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " + sDim.Render("On the command line:") + "\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni @ --patroni @ --no-mirror --no-node --no-model" - case cServe: - item = "\n Takes a GGUF model from Hugging Face to a replica that answers:\n the mirror fetches it, its checksum and memory are checked, it joins\n the catalog, and a replica is started and asked to say hello.\n\n " + - sDim.Render("On the command line:") + "\n llmgw ctl model serve OWNER/REPO" case cUpgrade: item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION" diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go new file mode 100644 index 0000000..14e87ca --- /dev/null +++ b/internal/cli/tui_hf.go @@ -0,0 +1,193 @@ +package cli + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" +) + +// Serving from Hugging Face: the registry's GGUF models, trending or +// searched, then a file of the one chosen, then `llmgw ctl model serve` with +// it — which fetches it into the mirror, checks it, adds it and starts it. + +var hfSorts = []struct { + label string + sort hf.Sort +}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}} + +type hfState struct { + query string + sort int + models []hf.Model + err error +} + +type consoleHFMsg struct { + query string + sort int + models []hf.Model + err error +} + +type consoleRepoMsg struct { + repo hf.Repo + err error +} + +// hfClients keeps one client per registry, for the client's few minutes of +// remembered listings. +var hfClients sync.Map + +func (m consoleModel) registry() *hf.Client { + endpoint := "" + if cfg, err := config.Load(m.file); err == nil { + endpoint = cfg.Mirror.Registry + } + c, _ := hfClients.LoadOrStore(endpoint, hf.New(endpoint, 20*time.Second)) + return c.(*hf.Client) +} + +func (m consoleModel) fetchHF() tea.Cmd { + c, q, sort := m.registry(), m.hf.query, m.hf.sort + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + got, err := c.Explore(ctx, hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50}) + return consoleHFMsg{query: q, sort: sort, models: got, err: err} + } +} + +func (m *consoleModel) fetchRepo(id string) tea.Cmd { + m.busy = "Reading " + id + c := m.registry() + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + r, err := c.Repo(ctx, id) + return consoleRepoMsg{r, err} + } +} + +// repoMenu offers a repository's GGUF files, smallest first as listed. +func (m consoleModel) repoMenu(r hf.Repo) *menuDialog { + var items []menuItem + for _, set := range hf.Group(r.Files) { + if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") { + continue + } + low := strings.ToLower(set.Name) + // Projectors and imatrix data are not a model to serve on their own. + if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") { + continue + } + label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes)) + if set.Shards > 1 { + label += fmt.Sprintf(" %d parts", set.Shards) + } + set := set + items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd { + m.serveForm(r, set) + return nil + }}) + } + title := r.ID + if len(items) == 0 { + items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}} + } + if r.Gated.Is() { + title += " · gated" + } + return &menuDialog{title: title, items: items} +} + +func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { + server := "--server=" + m.client.base + note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes)) + if r.Architecture != "" { + note += ", " + r.Architecture + } + if r.License != "" { + note += ", licence " + r.License + } + note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes." + m.openForm(&formDialog{title: "Serve " + r.ID, note: note, + fields: []formField{ + textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("replicas", "1", "0 runs none until it is first asked for"), + textF("context length", "4096", "what one request may use"), + textF("description", "", "optional: shown to clients in /v1/models"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + n, err := strconv.Atoi(v[1]) + if err != nil || n < 0 { + return nil, fmt.Errorf("replicas is a whole number") + } + if _, err := strconv.Atoi(v[2]); err != nil { + return nil, fmt.Errorf("the context length is a whole number") + } + args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name, + "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} + if n == 0 { + args = append(args, "--on-demand") + } + if v[3] != "" { + args = append(args, "--description="+v[3]) + } + return m.runAfter(args...), nil + }}) +} + +func (m *consoleModel) hfSearchDialog() { + m.openForm(&formDialog{title: "Search Hugging Face", + note: "GGUF models, which is what the nodes run.", + fields: []formField{textF("search", m.hf.query, "like qwen2.5 coder; empty lists them all")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + m.hf.query, m.hf.models, m.hf.err, m.row = v[0], nil, nil, 0 + return m.fetchHF(), nil + }}) +} + +func (m consoleModel) hfPanel(pw int) (string, string, string) { + keys := "→ or Enter open · ↑↓ choose · Q leave" + if m.inPanel { + keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave" + } + what := hfSorts[m.hf.sort].label + if m.hf.query != "" { + what = "“" + m.hf.query + "”, " + what + } + head := "\n GGUF models on Hugging Face: " + what + "\n\n" + switch { + case m.hf.err != nil: + return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) + + "\n\n " + sDim.Render("mirror.registry in server.yaml names another registry, for a site without the internet."), keys + case m.hf.models == nil: + return "Serve from Hugging Face", head + " " + m.spin.View() + " asking the registry…", keys + } + var rows [][]string + for _, x := range m.hf.models { + params := "" + if p := x.Params(); p > 0 { + params = fmt.Sprintf("%.1fB", float64(p)/1e9) + } + gated := "" + if x.Gated.Is() { + gated = sWarn.Render("gated") + } + rows = append(rows, []string{x.ID, params, x.Architecture(), compactNum(x.Downloads), compactNum(x.Likes), + ago(x.LastModified, time.Now()), gated}) + } + if len(rows) == 0 { + return "Serve from Hugging Face", head + " " + sDim.Render("Nothing found. / searches again."), keys + } + return "Serve from Hugging Face", head + table([]string{"REPOSITORY", "SIZE", "ARCH", "DOWNLOADS", "LIKES", "UPDATED", ""}, + rows, m.row, pw, m.inPanel), keys +}