Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
222 lines
6.8 KiB
Go
222 lines
6.8 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
)
|
|
|
|
// Serving from Hugging Face: the registry's GGUF models, trending or
|
|
// searched, then a file of the one chosen, then `llmgw ctl model serve` with
|
|
// it — which fetches it into the mirror, checks it, adds it and starts it.
|
|
|
|
var hfSorts = []struct {
|
|
label string
|
|
sort hf.Sort
|
|
}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}}
|
|
|
|
type hfState struct {
|
|
query string
|
|
sort int
|
|
models []hf.Model
|
|
err error
|
|
}
|
|
|
|
type consoleHFMsg struct {
|
|
query string
|
|
sort int
|
|
models []hf.Model
|
|
err error
|
|
}
|
|
|
|
type consoleRepoMsg struct {
|
|
repo hf.Repo
|
|
err error
|
|
}
|
|
|
|
// hfClients keeps one client per registry, for the client's few minutes of
|
|
// remembered listings.
|
|
var hfClients sync.Map
|
|
|
|
func (m consoleModel) registry() *hf.Client {
|
|
endpoint := ""
|
|
if cfg, err := config.Load(m.file); err == nil {
|
|
endpoint = cfg.Mirror.Registry
|
|
}
|
|
c, _ := hfClients.LoadOrStore(endpoint, hf.New(endpoint, 20*time.Second))
|
|
return c.(*hf.Client)
|
|
}
|
|
|
|
func (m consoleModel) fetchHF() tea.Cmd {
|
|
c, q, sort := m.registry(), m.hf.query, m.hf.sort
|
|
return func() tea.Msg {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
|
defer cancel()
|
|
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50}
|
|
// Browsing: models that chat. A search finds the rest too, an
|
|
// embedding model included.
|
|
if q == "" {
|
|
hq.Task = "text-generation"
|
|
}
|
|
got, err := c.Explore(ctx, hq)
|
|
return consoleHFMsg{query: q, sort: sort, models: got, err: err}
|
|
}
|
|
}
|
|
|
|
func (m *consoleModel) fetchRepo(id string) tea.Cmd {
|
|
m.busy = "Reading " + id
|
|
c := m.registry()
|
|
return func() tea.Msg {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
|
defer cancel()
|
|
r, err := c.Repo(ctx, id)
|
|
return consoleRepoMsg{r, err}
|
|
}
|
|
}
|
|
|
|
// repoMenu offers a repository's GGUF files, smallest first as listed.
|
|
func (m consoleModel) repoMenu(r hf.Repo) *menuDialog {
|
|
var items []menuItem
|
|
for _, set := range hf.Group(r.Files) {
|
|
if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") {
|
|
continue
|
|
}
|
|
low := strings.ToLower(set.Name)
|
|
// Projectors and imatrix data are not a model to serve on their own.
|
|
if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") {
|
|
continue
|
|
}
|
|
label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes))
|
|
if set.Shards > 1 {
|
|
label += fmt.Sprintf(" %d parts", set.Shards)
|
|
}
|
|
set := set
|
|
items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd {
|
|
m.serveForm(r, set)
|
|
return nil
|
|
}})
|
|
}
|
|
title := r.ID
|
|
if len(items) == 0 {
|
|
items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}}
|
|
}
|
|
if r.Gated.Is() {
|
|
title += " · gated"
|
|
}
|
|
return &menuDialog{title: title, items: items}
|
|
}
|
|
|
|
func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
|
|
server := "--server=" + m.client.base
|
|
note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes))
|
|
if r.Architecture != "" {
|
|
note += ", " + r.Architecture
|
|
}
|
|
if r.License != "" {
|
|
note += ", licence " + r.License
|
|
}
|
|
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes."
|
|
m.openForm(&formDialog{title: "Serve " + r.ID, note: note,
|
|
fields: []formField{
|
|
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
|
textF("replicas", "1", "0 runs none until it is first asked for"),
|
|
textF("context length", "4096", "what one request may use"),
|
|
textF("description", "", "optional: shown to clients in /v1/models"),
|
|
},
|
|
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
|
n, err := strconv.Atoi(v[1])
|
|
if err != nil || n < 0 {
|
|
return nil, fmt.Errorf("replicas is a whole number")
|
|
}
|
|
if _, err := strconv.Atoi(v[2]); err != nil {
|
|
return nil, fmt.Errorf("the context length is a whole number")
|
|
}
|
|
args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name,
|
|
"--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
|
|
if n == 0 {
|
|
args = append(args, "--on-demand")
|
|
}
|
|
if v[3] != "" {
|
|
args = append(args, "--description="+v[3])
|
|
}
|
|
return m.runAfter(args...), nil
|
|
}})
|
|
}
|
|
|
|
func (m *consoleModel) hfSearchDialog() {
|
|
m.openForm(&formDialog{title: "Search Hugging Face",
|
|
note: "GGUF models, which is what the nodes run.",
|
|
fields: []formField{textF("search", m.hf.query, "like qwen2.5 coder; empty lists them all")},
|
|
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
|
m.hf.query, m.hf.models, m.hf.err, m.row = v[0], nil, nil, 0
|
|
return m.fetchHF(), nil
|
|
}})
|
|
}
|
|
|
|
func (m consoleModel) hfPanel(pw int) (string, string, string) {
|
|
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
|
if m.inPanel {
|
|
keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave"
|
|
}
|
|
what := hfSorts[m.hf.sort].label
|
|
if m.hf.query != "" {
|
|
what = "“" + m.hf.query + "”, " + what
|
|
}
|
|
if m.hf.query == "" {
|
|
what = "text generation, " + what
|
|
}
|
|
head := "\n GGUF models on Hugging Face: " + what + "\n\n"
|
|
switch {
|
|
case m.hf.err != nil:
|
|
return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) +
|
|
"\n\n " + sDim.Render("mirror.registry in server.yaml names another registry, for a site without the internet."), keys
|
|
case m.hf.models == nil:
|
|
return "Serve from Hugging Face", head + " " + m.spin.View() + " asking the registry…", keys
|
|
}
|
|
var rows [][]string
|
|
for _, x := range m.hf.models {
|
|
params := ""
|
|
if p := x.Params(); p > 0 {
|
|
params = fmt.Sprintf("%.1fB", float64(p)/1e9)
|
|
}
|
|
gated := ""
|
|
if x.Gated.Is() {
|
|
gated = sWarn.Render("gated")
|
|
}
|
|
rows = append(rows, []string{x.ID, params, x.Architecture(), compactNum(x.Downloads), compactNum(x.Likes),
|
|
ageShort(x.LastModified), gated})
|
|
}
|
|
if len(rows) == 0 {
|
|
return "Serve from Hugging Face", head + " " + sDim.Render("Nothing found. / searches again."), keys
|
|
}
|
|
return "Serve from Hugging Face", head + table([]string{"REPOSITORY", "SIZE", "ARCH", "DOWNLOADS", "LIKES", "UPDATED", ""},
|
|
rows, m.row, pw, m.inPanel), keys
|
|
}
|
|
|
|
// ageShort is how long ago, in the largest unit that says it: 5m, 3h, 12d, 7mo, 2y.
|
|
func ageShort(t time.Time) string {
|
|
if t.IsZero() {
|
|
return ""
|
|
}
|
|
d := time.Since(t)
|
|
switch {
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm ago", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%dh ago", int(d.Hours()))
|
|
case d < 60*24*time.Hour:
|
|
return fmt.Sprintf("%dd ago", int(d.Hours()/24))
|
|
case d < 2*365*24*time.Hour:
|
|
return fmt.Sprintf("%dmo ago", int(d.Hours()/24/30))
|
|
}
|
|
return fmt.Sprintf("%dy ago", int(d.Hours()/24/365))
|
|
}
|