Serve a model is a view of the registry's GGUF models, trending or searched; Enter on one lists its files, and the one chosen is served with llmgw ctl model serve. A form of one field is applied by Enter on it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
6a7b4bb6a0
commit
2bf43ed90c
5 files changed
+300
-31
No files matched your search
+5
-2
@@ -43,8 +43,11 @@ what can be done to it, as forms and questions:
|
||||
issue (shown once), limits, revoke; organizations, teams and projects — add,
|
||||
rename, move, remove, limits for the level and for each member, capacity
|
||||
quotas, how long conversations and agent notes are kept.
|
||||
- **Do**: serve a model from Hugging Face, upgrade the cluster, restart every
|
||||
replica, back up now.
|
||||
- **Serve a model** (new view): GGUF models on Hugging Face — trending, most
|
||||
downloaded, most liked or newest, or searched with / — then one of the
|
||||
chosen repository's files, a name and a replica count, and it is fetched,
|
||||
checked, added and started. The registry is mirror.registry's when set.
|
||||
- **Do**: upgrade the cluster, restart every replica, back up now.
|
||||
|
||||
The people side runs the same commands as `llmgw users`, `keys` and `org`,
|
||||
against the same store, so their checks and messages are the ones shown. A
|
||||
|
||||
@@ -144,7 +144,8 @@ func (m consoleModel) actionKey(k tea.KeyMsg) (tea.Model, tea.Cmd, bool) {
|
||||
fl.opt = (fl.opt + d) % len(fl.options)
|
||||
}
|
||||
case tea.KeyEnter:
|
||||
if f.focus < len(f.fields) {
|
||||
// A form of one field is applied by Enter on it.
|
||||
if f.focus < len(f.fields) && len(f.fields) > 1 {
|
||||
f.fields[f.focus].text.Blur()
|
||||
f.focus++
|
||||
if f.focus < len(f.fields) {
|
||||
@@ -635,30 +636,6 @@ func (m *consoleModel) runAfter(args ...string) tea.Cmd {
|
||||
func (m *consoleModel) doEntry() tea.Cmd {
|
||||
server := "--server=" + m.client.base
|
||||
switch m.menu {
|
||||
case cServe:
|
||||
m.openForm(&formDialog{title: "Serve a model",
|
||||
note: "A GGUF repository on Hugging Face, like bartowski/Qwen2.5-3B-Instruct-GGUF.",
|
||||
fields: []formField{
|
||||
textF("repository", "", "OWNER/REPO"),
|
||||
textF("quantisation", "Q4_K_M", ""),
|
||||
textF("name", "", "in the catalog; empty takes it from the repository"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "4096", "what one request may use"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if !strings.Contains(v[0], "/") {
|
||||
return nil, fmt.Errorf("the repository is OWNER/REPO")
|
||||
}
|
||||
args := []string{"ctl", server, "model", "serve", v[0], "--quant=" + v[1], "--replicas=" + v[3],
|
||||
"--context-length=" + v[4]}
|
||||
if v[3] == "0" {
|
||||
args = append(args, "--on-demand")
|
||||
}
|
||||
if v[2] != "" {
|
||||
args = append(args, "--name="+v[2])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
case cUpgrade:
|
||||
m.openForm(&formDialog{title: "Upgrade the cluster",
|
||||
note: "This machine runs " + version.Version + ".",
|
||||
|
||||
@@ -257,3 +257,65 @@ func TestTheSidebarFitsAShortScreen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
|
||||
reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/api/models":
|
||||
if r.URL.Query().Get("search") != "qwen" || r.URL.Query().Get("filter") != "gguf" {
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","downloads":1200,"gguf":{"total":3000000000,"architecture":"qwen2"}}]`))
|
||||
case r.URL.Path == "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF":
|
||||
_, _ = w.Write([]byte(`{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","sha":"abc"}`))
|
||||
case strings.HasPrefix(r.URL.Path, "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF/tree/"):
|
||||
_, _ = w.Write([]byte(`[{"type":"file","path":"Qwen2.5-3B-Instruct-Q4_K_M.gguf","lfs":{"oid":"x","size":2000000000}},
|
||||
{"type":"file","path":"Qwen2.5-3B-Instruct-Q8_0.gguf","lfs":{"oid":"y","size":3000000000}},
|
||||
{"type":"file","path":"mmproj-f16.gguf","size":10}]`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer reg.Close()
|
||||
dir := t.TempDir()
|
||||
cfgPath := dir + "/server.yaml"
|
||||
if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := newConsoleModel("127.0.0.1:9090", cfgPath)
|
||||
m.loaded, m.menu, m.inPanel = true, cServe, true
|
||||
var tm tea.Model = m
|
||||
|
||||
run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
|
||||
if cmd != nil {
|
||||
tm, _ = tm.Update(cmd())
|
||||
}
|
||||
return tm
|
||||
}
|
||||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("/")})
|
||||
tm = press(tm, "q", "w", "e", "n")
|
||||
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
tm = run(tm, cmd)
|
||||
if v := tm.View(); !strings.Contains(v, "bartowski/Qwen2.5-3B-Instruct-GGUF") || !strings.Contains(v, "3.0B") {
|
||||
t.Fatalf("no result:\n%s", v)
|
||||
}
|
||||
tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
tm = run(tm, cmd)
|
||||
d := tm.(consoleModel).menuD
|
||||
if d == nil || len(d.items) != 2 {
|
||||
t.Fatalf("files offered: %+v", d)
|
||||
}
|
||||
tm = press(tm, "down", "enter")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" {
|
||||
t.Fatalf("form: %+v", f)
|
||||
}
|
||||
f.focus = len(f.fields)
|
||||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
act := tm.(consoleModel).action
|
||||
got := strings.Join(act.targets, " ")
|
||||
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") {
|
||||
t.Fatalf("would run %q %q", act.kind, got)
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
@@ -151,6 +152,7 @@ type consoleModel struct {
|
||||
st *storeHolder
|
||||
people peopleData
|
||||
mirror consoleMirrorMsg
|
||||
hf hfState
|
||||
}
|
||||
|
||||
var usageGroupings = []string{"team", "project", "org"}
|
||||
@@ -286,6 +288,21 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.logs = msg
|
||||
case consoleMirrorMsg:
|
||||
m.mirror = msg
|
||||
case consoleHFMsg:
|
||||
if msg.query == m.hf.query && msg.sort == m.hf.sort {
|
||||
m.hf.models, m.hf.err = msg.models, msg.err
|
||||
if m.hf.models == nil && msg.err == nil {
|
||||
m.hf.models = []hf.Model{}
|
||||
}
|
||||
}
|
||||
case consoleRepoMsg:
|
||||
m.busy = ""
|
||||
if msg.err != nil {
|
||||
m.dialog, m.dialogTitle = "error", "Hugging Face"
|
||||
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
|
||||
} else {
|
||||
m.menuD = m.repoMenu(msg.repo)
|
||||
}
|
||||
case consolePeopleMsg:
|
||||
m.people = peopleData(msg)
|
||||
case consoleQuotasMsg:
|
||||
@@ -366,7 +383,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
return m.ask("Replicate the store", "user@address of database machine", 2)
|
||||
case cLeave:
|
||||
return m.leave()
|
||||
case cServe, cUpgrade, cRestartAll, cBackup:
|
||||
case cUpgrade, cRestartAll, cBackup:
|
||||
return m, m.doEntry()
|
||||
case cOverview:
|
||||
return m, nil
|
||||
@@ -408,6 +425,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.menuD = m.quotaMenu(m.quotas[m.row])
|
||||
case cMirror:
|
||||
m.menuD = m.mirrorMenu(m.mirror.res.Models[m.row])
|
||||
case cServe:
|
||||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||||
case cAccounts:
|
||||
m.menuD = m.userMenu(m.people.users[m.row])
|
||||
case cKeys:
|
||||
@@ -415,6 +434,16 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cOrgs:
|
||||
m.menuD = m.orgMenu(m.people.orgs[m.row])
|
||||
}
|
||||
case "/":
|
||||
if m.menu == cServe {
|
||||
m.hfSearchDialog()
|
||||
}
|
||||
case "s":
|
||||
if m.menu == cServe {
|
||||
m.hf.sort = (m.hf.sort + 1) % len(hfSorts)
|
||||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||||
return m, m.fetchHF()
|
||||
}
|
||||
case "a":
|
||||
switch m.menu {
|
||||
case cQuotas:
|
||||
@@ -456,6 +485,10 @@ func (m consoleModel) onEnter() tea.Cmd {
|
||||
return m.fetchQuotas()
|
||||
case cMirror:
|
||||
return m.fetchMirror()
|
||||
case cServe:
|
||||
if m.hf.models == nil {
|
||||
return m.fetchHF()
|
||||
}
|
||||
case cAccounts, cKeys, cOrgs:
|
||||
// The quotas too, for an organization's quota form.
|
||||
return tea.Batch(m.fetchPeople(), m.fetchQuotas())
|
||||
@@ -766,6 +799,8 @@ func (m consoleModel) rowCount() int {
|
||||
return len(m.quotas)
|
||||
case cMirror:
|
||||
return len(m.mirror.res.Models)
|
||||
case cServe:
|
||||
return len(m.hf.models)
|
||||
case cAccounts:
|
||||
return len(m.people.users)
|
||||
case cKeys:
|
||||
@@ -966,6 +1001,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
return m.peoplePanel(pw)
|
||||
case cMirror:
|
||||
return m.mirrorPanel(pw)
|
||||
case cServe:
|
||||
return m.hfPanel(pw)
|
||||
case cSettings:
|
||||
return m.settingsPanel(pw)
|
||||
}
|
||||
@@ -978,9 +1015,6 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
|
||||
sDim.Render("On the command line:") +
|
||||
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
|
||||
case cServe:
|
||||
item = "\n Takes a GGUF model from Hugging Face to a replica that answers:\n the mirror fetches it, its checksum and memory are checked, it joins\n the catalog, and a replica is started and asked to say hello.\n\n " +
|
||||
sDim.Render("On the command line:") + "\n llmgw ctl model serve OWNER/REPO"
|
||||
case cUpgrade:
|
||||
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
|
||||
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
)
|
||||
|
||||
// Serving from Hugging Face: the registry's GGUF models, trending or
|
||||
// searched, then a file of the one chosen, then `llmgw ctl model serve` with
|
||||
// it — which fetches it into the mirror, checks it, adds it and starts it.
|
||||
|
||||
var hfSorts = []struct {
|
||||
label string
|
||||
sort hf.Sort
|
||||
}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}}
|
||||
|
||||
type hfState struct {
|
||||
query string
|
||||
sort int
|
||||
models []hf.Model
|
||||
err error
|
||||
}
|
||||
|
||||
type consoleHFMsg struct {
|
||||
query string
|
||||
sort int
|
||||
models []hf.Model
|
||||
err error
|
||||
}
|
||||
|
||||
type consoleRepoMsg struct {
|
||||
repo hf.Repo
|
||||
err error
|
||||
}
|
||||
|
||||
// hfClients keeps one client per registry, for the client's few minutes of
|
||||
// remembered listings.
|
||||
var hfClients sync.Map
|
||||
|
||||
func (m consoleModel) registry() *hf.Client {
|
||||
endpoint := ""
|
||||
if cfg, err := config.Load(m.file); err == nil {
|
||||
endpoint = cfg.Mirror.Registry
|
||||
}
|
||||
c, _ := hfClients.LoadOrStore(endpoint, hf.New(endpoint, 20*time.Second))
|
||||
return c.(*hf.Client)
|
||||
}
|
||||
|
||||
func (m consoleModel) fetchHF() tea.Cmd {
|
||||
c, q, sort := m.registry(), m.hf.query, m.hf.sort
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
got, err := c.Explore(ctx, hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50})
|
||||
return consoleHFMsg{query: q, sort: sort, models: got, err: err}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *consoleModel) fetchRepo(id string) tea.Cmd {
|
||||
m.busy = "Reading " + id
|
||||
c := m.registry()
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
r, err := c.Repo(ctx, id)
|
||||
return consoleRepoMsg{r, err}
|
||||
}
|
||||
}
|
||||
|
||||
// repoMenu offers a repository's GGUF files, smallest first as listed.
|
||||
func (m consoleModel) repoMenu(r hf.Repo) *menuDialog {
|
||||
var items []menuItem
|
||||
for _, set := range hf.Group(r.Files) {
|
||||
if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") {
|
||||
continue
|
||||
}
|
||||
low := strings.ToLower(set.Name)
|
||||
// Projectors and imatrix data are not a model to serve on their own.
|
||||
if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") {
|
||||
continue
|
||||
}
|
||||
label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes))
|
||||
if set.Shards > 1 {
|
||||
label += fmt.Sprintf(" %d parts", set.Shards)
|
||||
}
|
||||
set := set
|
||||
items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd {
|
||||
m.serveForm(r, set)
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
title := r.ID
|
||||
if len(items) == 0 {
|
||||
items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}}
|
||||
}
|
||||
if r.Gated.Is() {
|
||||
title += " · gated"
|
||||
}
|
||||
return &menuDialog{title: title, items: items}
|
||||
}
|
||||
|
||||
func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
|
||||
server := "--server=" + m.client.base
|
||||
note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes))
|
||||
if r.Architecture != "" {
|
||||
note += ", " + r.Architecture
|
||||
}
|
||||
if r.License != "" {
|
||||
note += ", licence " + r.License
|
||||
}
|
||||
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes."
|
||||
m.openForm(&formDialog{title: "Serve " + r.ID, note: note,
|
||||
fields: []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "4096", "what one request may use"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 0 {
|
||||
return nil, fmt.Errorf("replicas is a whole number")
|
||||
}
|
||||
if _, err := strconv.Atoi(v[2]); err != nil {
|
||||
return nil, fmt.Errorf("the context length is a whole number")
|
||||
}
|
||||
args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name,
|
||||
"--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
|
||||
if n == 0 {
|
||||
args = append(args, "--on-demand")
|
||||
}
|
||||
if v[3] != "" {
|
||||
args = append(args, "--description="+v[3])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m *consoleModel) hfSearchDialog() {
|
||||
m.openForm(&formDialog{title: "Search Hugging Face",
|
||||
note: "GGUF models, which is what the nodes run.",
|
||||
fields: []formField{textF("search", m.hf.query, "like qwen2.5 coder; empty lists them all")},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
m.hf.query, m.hf.models, m.hf.err, m.row = v[0], nil, nil, 0
|
||||
return m.fetchHF(), nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m consoleModel) hfPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave"
|
||||
}
|
||||
what := hfSorts[m.hf.sort].label
|
||||
if m.hf.query != "" {
|
||||
what = "“" + m.hf.query + "”, " + what
|
||||
}
|
||||
head := "\n GGUF models on Hugging Face: " + what + "\n\n"
|
||||
switch {
|
||||
case m.hf.err != nil:
|
||||
return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) +
|
||||
"\n\n " + sDim.Render("mirror.registry in server.yaml names another registry, for a site without the internet."), keys
|
||||
case m.hf.models == nil:
|
||||
return "Serve from Hugging Face", head + " " + m.spin.View() + " asking the registry…", keys
|
||||
}
|
||||
var rows [][]string
|
||||
for _, x := range m.hf.models {
|
||||
params := ""
|
||||
if p := x.Params(); p > 0 {
|
||||
params = fmt.Sprintf("%.1fB", float64(p)/1e9)
|
||||
}
|
||||
gated := ""
|
||||
if x.Gated.Is() {
|
||||
gated = sWarn.Render("gated")
|
||||
}
|
||||
rows = append(rows, []string{x.ID, params, x.Architecture(), compactNum(x.Downloads), compactNum(x.Likes),
|
||||
ago(x.LastModified, time.Now()), gated})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Serve from Hugging Face", head + " " + sDim.Render("Nothing found. / searches again."), keys
|
||||
}
|
||||
return "Serve from Hugging Face", head + table([]string{"REPOSITORY", "SIZE", "ARCH", "DOWNLOADS", "LIKES", "UPDATED", ""},
|
||||
rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
Reference in new issue
Block a user