Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+88 -15
View File
@@ -11,6 +11,7 @@ import (
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
)
@@ -24,15 +25,19 @@ var hfSorts = []struct {
}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}}
type hfState struct {
query string
sort int
models []hf.Model
err error
query string
sort int
// safetensors lists original weights, for vLLM on GPUs, rather than
// GGUF files for llama.cpp.
safetensors bool
models []hf.Model
err error
}
type consoleHFMsg struct {
query string
sort int
st bool
models []hf.Model
err error
}
@@ -56,18 +61,21 @@ func (m consoleModel) registry() *hf.Client {
}
func (m consoleModel) fetchHF() tea.Cmd {
c, q, sort := m.registry(), m.hf.query, m.hf.sort
c, q, sort, st := m.registry(), m.hf.query, m.hf.sort, m.hf.safetensors
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
defer cancel()
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50}
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: !st, Limit: 50}
if st {
hq.Filters = []string{"safetensors"}
}
// Browsing: models that chat. A search finds the rest too, an
// embedding model included.
if q == "" {
hq.Task = "text-generation"
}
got, err := c.Explore(ctx, hq)
return consoleHFMsg{query: q, sort: sort, models: got, err: err}
return consoleHFMsg{query: q, sort: sort, st: st, models: got, err: err}
}
}
@@ -85,6 +93,17 @@ func (m *consoleModel) fetchRepo(id string) tea.Cmd {
// repoMenu offers a repository's GGUF files, smallest first as listed.
func (m consoleModel) repoMenu(r hf.Repo) *menuDialog {
var items []menuItem
if m.hf.safetensors {
title := r.ID
set, err := modelserve.SafetensorsSet(r)
if err != nil {
return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}}
}
return &menuDialog{title: title, items: []menuItem{{
fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)),
func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil },
}}}
}
for _, set := range hf.Group(r.Files) {
if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") {
continue
@@ -124,13 +143,19 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
note += ", licence " + r.License
}
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes."
fields := []formField{
textF("name", hf.ModelName(r.ID), "what clients ask for"),
textF("replicas", "1", "0 runs none until it is first asked for"),
textF("context length", "4096", "what one request may use"),
textF("description", "", "optional: shown to clients in /v1/models"),
}
// A vision model: its projector, loaded beside it, is what reads images.
if proj := modelserve.Projectors(r); len(proj) > 0 {
fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0]))
fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model"
}
m.openForm(&formDialog{title: "Serve " + r.ID, note: note,
fields: []formField{
textF("name", hf.ModelName(r.ID), "what clients ask for"),
textF("replicas", "1", "0 runs none until it is first asked for"),
textF("context length", "4096", "what one request may use"),
textF("description", "", "optional: shown to clients in /v1/models"),
},
fields: fields,
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
n, err := strconv.Atoi(v[1])
if err != nil || n < 0 {
@@ -147,6 +172,50 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
if v[3] != "" {
args = append(args, "--description="+v[3])
}
if len(v) > 4 && !strings.HasPrefix(v[4], "none") {
args = append(args, "--projector="+v[4])
}
return m.runAfter(args...), nil
}})
}
func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) {
server := "--server=" + m.client.base
var classes []string
for _, g := range m.data.sum.GPUs {
classes = append(classes, g.Class)
}
if len(classes) == 0 {
m.dialog, m.dialogTitle = "error", "No GPUs"
m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" +
"F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key")
return
}
m.openForm(&formDialog{title: "Serve " + r.ID,
note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+
"Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)),
fields: []formField{
textF("name", hf.ModelName(r.ID), "what clients ask for"),
textF("replicas", "1", "0 runs none until it is first asked for"),
textF("context length", "8192", "what one request may use; vLLM's --max-model-len"),
textF("description", "", "optional: shown to clients in /v1/models"),
choiceF("GPU class", classes, classes[0]),
textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"),
},
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} {
if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) {
return nil, fmt.Errorf("%s is a whole number", what)
}
}
args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4],
"--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
if v[1] == "0" {
args = append(args, "--on-demand")
}
if v[3] != "" {
args = append(args, "--description="+v[3])
}
return m.runAfter(args...), nil
}})
}
@@ -164,7 +233,7 @@ func (m *consoleModel) hfSearchDialog() {
func (m consoleModel) hfPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave"
keys = "↑↓ row · Enter choose · / search · S order · F GGUF or original weights · ← back · Q leave"
}
what := hfSorts[m.hf.sort].label
if m.hf.query != "" {
@@ -173,7 +242,11 @@ func (m consoleModel) hfPanel(pw int) (string, string, string) {
if m.hf.query == "" {
what = "text generation, " + what
}
head := "\n GGUF models on Hugging Face: " + what + "\n\n"
kind := "GGUF models, for llama.cpp"
if m.hf.safetensors {
kind = "Original weights, for vLLM on GPUs"
}
head := "\n " + kind + " on Hugging Face: " + what + "\n\n"
switch {
case m.hf.err != nil:
return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) +