The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
327 lines
10 KiB
Go
327 lines
10 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
|
)
|
|
|
|
// Serving from Hugging Face: the registry's GGUF models, trending or
|
|
// searched, then a file of the one chosen, then `llmgw ctl model serve` with
|
|
// it — which fetches it into the mirror, checks it, adds it and starts it.
|
|
|
|
var hfSorts = []struct {
|
|
label string
|
|
sort hf.Sort
|
|
}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}}
|
|
|
|
type hfState struct {
|
|
query string
|
|
sort int
|
|
// safetensors lists original weights, for vLLM on GPUs, rather than
|
|
// GGUF files for llama.cpp.
|
|
safetensors bool
|
|
models []hf.Model
|
|
err error
|
|
}
|
|
|
|
type consoleHFMsg struct {
|
|
query string
|
|
sort int
|
|
st bool
|
|
models []hf.Model
|
|
err error
|
|
}
|
|
|
|
type consoleRepoMsg struct {
|
|
plan modelserve.Plan
|
|
err error
|
|
}
|
|
|
|
// hfClients keeps one client per registry, for the client's few minutes of
|
|
// remembered listings.
|
|
var hfClients sync.Map
|
|
|
|
func (m consoleModel) registry() *hf.Client {
|
|
endpoint := ""
|
|
if cfg, err := config.Load(m.file); err == nil {
|
|
endpoint = cfg.Mirror.Registry
|
|
}
|
|
c, _ := hfClients.LoadOrStore(endpoint, hf.New(endpoint, 20*time.Second))
|
|
return c.(*hf.Client)
|
|
}
|
|
|
|
func (m consoleModel) fetchHF() tea.Cmd {
|
|
c, q, sort, st := m.registry(), m.hf.query, m.hf.sort, m.hf.safetensors
|
|
return func() tea.Msg {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
|
defer cancel()
|
|
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: !st, Limit: 50}
|
|
if st {
|
|
hq.Filters = []string{"safetensors"}
|
|
}
|
|
// Browsing: models that chat. A search finds the rest too, an
|
|
// embedding model included.
|
|
if q == "" {
|
|
hq.Task = "text-generation"
|
|
}
|
|
got, err := c.Explore(ctx, hq)
|
|
return consoleHFMsg{query: q, sort: sort, st: st, models: got, err: err}
|
|
}
|
|
}
|
|
|
|
// fetchRepo asks the control plane for a repository's plan: its files, and
|
|
// how each could run on this cluster.
|
|
func (m *consoleModel) fetchRepo(id string) tea.Cmd {
|
|
m.busy = "Planning " + id
|
|
c := m.client
|
|
format := "gguf"
|
|
if m.hf.safetensors {
|
|
format = "safetensors"
|
|
}
|
|
return func() tea.Msg {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
var p modelserve.Plan
|
|
err := c.get(ctx, "/api/v1/plan?"+url.Values{"repo": {id}, "format": {format}}.Encode(), &p)
|
|
return consoleRepoMsg{p, err}
|
|
}
|
|
}
|
|
|
|
// optionLabel is one way to run a file, with what it needs and whether the
|
|
// hardware is free.
|
|
func optionLabel(o modelplan.Option) string {
|
|
now := sDim.Render("busy now")
|
|
if o.FreeNow {
|
|
now = sOK.Render("free")
|
|
}
|
|
need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB)
|
|
if o.CPU {
|
|
need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB)
|
|
}
|
|
return fmt.Sprintf("%s · %s · %s", o.Label(), need, now)
|
|
}
|
|
|
|
// repoMenu offers a repository's files, each with the best way it runs here;
|
|
// the plan's choice first.
|
|
func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog {
|
|
title := p.Repo
|
|
if p.Gated {
|
|
title += " · gated"
|
|
}
|
|
var items []menuItem
|
|
if !p.Servable {
|
|
return &menuDialog{title: title, items: []menuItem{{"The nodes' engine cannot run it: " + p.Reason,
|
|
func(*consoleModel) tea.Cmd { return nil }}}}
|
|
}
|
|
rec, hasRec := p.Recommended()
|
|
arts := append([]modelserve.PlannedArtifact{}, p.Artifacts...)
|
|
// The recommended file first, then the rest as the repository lists them.
|
|
for i, a := range arts {
|
|
if hasRec && a.Name == rec.Artifact && i > 0 {
|
|
arts = append([]modelserve.PlannedArtifact{a}, append(arts[:i:i], arts[i+1:]...)...)
|
|
break
|
|
}
|
|
}
|
|
for _, a := range arts {
|
|
name := a.Name
|
|
if a.Format == "safetensors" {
|
|
name = "original weights"
|
|
}
|
|
label := fmt.Sprintf("%-40s %8s ", truncate(name, 40), bytesShort(a.Bytes))
|
|
switch {
|
|
case a.Best == nil:
|
|
label += sBad.Render("fits nowhere")
|
|
default:
|
|
label += optionLabel(*a.Best)
|
|
}
|
|
if hasRec && a.Name == rec.Artifact {
|
|
label = sOK.Render("★ ") + label
|
|
} else {
|
|
label = " " + label
|
|
}
|
|
a := a
|
|
items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd {
|
|
if a.Best == nil {
|
|
m.dialog, m.dialogTitle = "error", a.Name
|
|
m.dialogText = fmt.Sprintf("No card holds %s with a %d-token context, nor does any node's memory.\n"+
|
|
"A smaller quantisation, or more cards free.\n\n%s", a.Name, p.Context, sDim.Render("any key"))
|
|
return nil
|
|
}
|
|
m.serveForm(p, a)
|
|
return nil
|
|
}})
|
|
}
|
|
return &menuDialog{title: title, items: items}
|
|
}
|
|
|
|
func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) {
|
|
server := "--server=" + m.client.base
|
|
var opts []modelplan.Option
|
|
for _, o := range p.Options {
|
|
if o.Artifact == a.Name {
|
|
opts = append(opts, o)
|
|
}
|
|
}
|
|
var where []string
|
|
for _, o := range opts {
|
|
where = append(where, optionLabel(o))
|
|
}
|
|
note := fmt.Sprintf("%s, %s", a.Name, bytesShort(a.Bytes))
|
|
if a.Format == "safetensors" {
|
|
note = fmt.Sprintf("Original weights, %s in %d files with their config and tokenizer, for vLLM", bytesShort(a.Bytes), a.Shards)
|
|
}
|
|
if p.Arch != "" {
|
|
note += ", " + p.Arch
|
|
}
|
|
if p.License != "" {
|
|
note += ", licence " + p.License
|
|
}
|
|
if p.Note != "" {
|
|
note += "\n" + sWarn.Render("! ") + p.Note
|
|
}
|
|
note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+
|
|
"catalog, and a replica is started and asked to say hello.", p.Context)
|
|
ctxDefault := "4096"
|
|
if a.Format == "safetensors" {
|
|
ctxDefault = strconv.Itoa(p.Context)
|
|
}
|
|
fields := []formField{
|
|
textF("name", hf.ModelName(p.Repo), "what clients ask for"),
|
|
textF("replicas", "1", "0 runs none until it is first asked for"),
|
|
textF("context length", ctxDefault, "what one request may use"),
|
|
textF("description", "", "optional: shown to clients in /v1/models"),
|
|
choiceF("runs on", where, where[0]),
|
|
choiceF("when that is busy", []string{"the next best way", "wait for it"}, "the next best way"),
|
|
}
|
|
fields[4].hint = "every way this file fits, best first"
|
|
proj := -1
|
|
if a.Format == "gguf" && len(p.Projectors) > 0 {
|
|
fields = append(fields, choiceF("images", append(append([]string{}, p.Projectors...), "none: text only"), p.Projectors[0]))
|
|
fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model"
|
|
proj = len(fields) - 1
|
|
}
|
|
m.openForm(&formDialog{title: "Serve " + p.Repo, note: note, fields: fields,
|
|
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
|
n, err := strconv.Atoi(v[1])
|
|
if err != nil || n < 0 {
|
|
return nil, fmt.Errorf("replicas is a whole number")
|
|
}
|
|
if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 {
|
|
return nil, fmt.Errorf("the context length is a whole number")
|
|
}
|
|
args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1],
|
|
"--context-length=" + v[2]}
|
|
if a.Format == "safetensors" {
|
|
args = append(args, "--format=safetensors")
|
|
} else {
|
|
args = append(args, "--file="+a.Name)
|
|
}
|
|
o := opts[slices.Index(where, v[4])]
|
|
if o.CPU {
|
|
args = append(args, "--cpu")
|
|
} else {
|
|
args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode),
|
|
"--nodes="+strconv.Itoa(max(o.Nodes, 1)))
|
|
}
|
|
if strings.HasPrefix(v[5], "the next") && len(opts) > 1 {
|
|
args = append(args, "--fallbacks")
|
|
}
|
|
if n == 0 {
|
|
args = append(args, "--on-demand")
|
|
}
|
|
if v[3] != "" {
|
|
args = append(args, "--description="+v[3])
|
|
}
|
|
if proj >= 0 && !strings.HasPrefix(v[proj], "none") {
|
|
args = append(args, "--projector="+v[proj])
|
|
}
|
|
return m.runAfter(args...), nil
|
|
}})
|
|
}
|
|
|
|
func (m *consoleModel) hfSearchDialog() {
|
|
m.openForm(&formDialog{title: "Search Hugging Face",
|
|
note: "GGUF models, which is what the nodes run.",
|
|
fields: []formField{textF("search", m.hf.query, "like qwen2.5 coder; empty lists them all")},
|
|
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
|
m.hf.query, m.hf.models, m.hf.err, m.row = v[0], nil, nil, 0
|
|
return m.fetchHF(), nil
|
|
}})
|
|
}
|
|
|
|
func (m consoleModel) hfPanel(pw int) (string, string, string) {
|
|
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
|
if m.inPanel {
|
|
keys = "↑↓ row · Enter choose · / search · S order · F GGUF or original weights · ← back · Q leave"
|
|
}
|
|
what := hfSorts[m.hf.sort].label
|
|
if m.hf.query != "" {
|
|
what = "“" + m.hf.query + "”, " + what
|
|
}
|
|
if m.hf.query == "" {
|
|
what = "text generation, " + what
|
|
}
|
|
kind := "GGUF models, for llama.cpp"
|
|
if m.hf.safetensors {
|
|
kind = "Original weights, for vLLM on GPUs"
|
|
}
|
|
head := "\n " + kind + " on Hugging Face: " + what + "\n\n"
|
|
switch {
|
|
case m.hf.err != nil:
|
|
return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) +
|
|
"\n\n " + sDim.Render("mirror.registry in server.yaml names another registry, for a site without the internet."), keys
|
|
case m.hf.models == nil:
|
|
return "Serve from Hugging Face", head + " " + m.spin.View() + " asking the registry…", keys
|
|
}
|
|
var rows [][]string
|
|
for _, x := range m.hf.models {
|
|
params := ""
|
|
if p := x.Params(); p > 0 {
|
|
params = fmt.Sprintf("%.1fB", float64(p)/1e9)
|
|
}
|
|
gated := ""
|
|
if x.Gated.Is() {
|
|
gated = sWarn.Render("gated")
|
|
}
|
|
rows = append(rows, []string{x.ID, params, x.Architecture(), compactNum(x.Downloads), compactNum(x.Likes),
|
|
ageShort(x.LastModified), gated})
|
|
}
|
|
if len(rows) == 0 {
|
|
return "Serve from Hugging Face", head + " " + sDim.Render("Nothing found. / searches again."), keys
|
|
}
|
|
return "Serve from Hugging Face", head + table([]string{"REPOSITORY", "SIZE", "ARCH", "DOWNLOADS", "LIKES", "UPDATED", ""},
|
|
rows, m.row, pw, m.inPanel), keys
|
|
}
|
|
|
|
// ageShort is how long ago, in the largest unit that says it: 5m, 3h, 12d, 7mo, 2y.
|
|
func ageShort(t time.Time) string {
|
|
if t.IsZero() {
|
|
return ""
|
|
}
|
|
d := time.Since(t)
|
|
switch {
|
|
case d < time.Hour:
|
|
return fmt.Sprintf("%dm ago", int(d.Minutes()))
|
|
case d < 48*time.Hour:
|
|
return fmt.Sprintf("%dh ago", int(d.Hours()))
|
|
case d < 60*24*time.Hour:
|
|
return fmt.Sprintf("%dd ago", int(d.Hours()/24))
|
|
case d < 2*365*24*time.Hour:
|
|
return fmt.Sprintf("%dmo ago", int(d.Hours()/24/30))
|
|
}
|
|
return fmt.Sprintf("%dy ago", int(d.Hours()/24/365))
|
|
}
|