Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
@@ -24,15 +25,27 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||
"A vision model's projector (--projector) is fetched beside it and loaded with\n" +
|
||||
"it. --format safetensors serves the original weights instead, with their\n" +
|
||||
"config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" +
|
||||
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand\n" +
|
||||
" llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Repo = args[0]
|
||||
if onDemand {
|
||||
req.Replicas = 0
|
||||
}
|
||||
// One kind of GPU in the fleet: that one, unless told.
|
||||
if req.Format == "safetensors" && req.GPUClass == "" {
|
||||
var sum adminapi.Summary
|
||||
if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 {
|
||||
req.GPUClass = sum.GPUs[0].Class
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
@@ -44,6 +57,10 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
|
||||
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)")
|
||||
f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
|
||||
@@ -292,7 +292,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
case consoleMirrorMsg:
|
||||
m.mirror = msg
|
||||
case consoleHFMsg:
|
||||
if msg.query == m.hf.query && msg.sort == m.hf.sort {
|
||||
if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors {
|
||||
m.hf.models, m.hf.err = msg.models, msg.err
|
||||
if m.hf.models == nil && msg.err == nil {
|
||||
m.hf.models = []hf.Model{}
|
||||
@@ -447,7 +447,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cQuotas:
|
||||
m.menuD = m.quotaMenu(m.quotas[m.row])
|
||||
case cMirror:
|
||||
m.menuD = m.mirrorMenu(m.mirror.res.Models[m.row])
|
||||
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
|
||||
case cServe:
|
||||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||||
case cAccounts:
|
||||
@@ -464,6 +464,12 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.menu == cServe {
|
||||
m.hfSearchDialog()
|
||||
}
|
||||
case "f":
|
||||
if m.menu == cServe {
|
||||
m.hf.safetensors = !m.hf.safetensors
|
||||
m.hf.models, m.hf.err, m.row = nil, nil, 0
|
||||
return m, m.fetchHF()
|
||||
}
|
||||
case "s":
|
||||
if m.menu == cServe {
|
||||
m.hf.sort = (m.hf.sort + 1) % len(hfSorts)
|
||||
@@ -824,7 +830,7 @@ func (m consoleModel) rowCount() int {
|
||||
case cQuotas:
|
||||
return len(m.quotas)
|
||||
case cMirror:
|
||||
return len(m.mirror.res.Models)
|
||||
return len(m.mirrorGroups())
|
||||
case cServe:
|
||||
return len(m.hf.models)
|
||||
case cAccounts:
|
||||
|
||||
+88
-15
@@ -11,6 +11,7 @@ import (
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
)
|
||||
|
||||
@@ -24,15 +25,19 @@ var hfSorts = []struct {
|
||||
}{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}}
|
||||
|
||||
type hfState struct {
|
||||
query string
|
||||
sort int
|
||||
models []hf.Model
|
||||
err error
|
||||
query string
|
||||
sort int
|
||||
// safetensors lists original weights, for vLLM on GPUs, rather than
|
||||
// GGUF files for llama.cpp.
|
||||
safetensors bool
|
||||
models []hf.Model
|
||||
err error
|
||||
}
|
||||
|
||||
type consoleHFMsg struct {
|
||||
query string
|
||||
sort int
|
||||
st bool
|
||||
models []hf.Model
|
||||
err error
|
||||
}
|
||||
@@ -56,18 +61,21 @@ func (m consoleModel) registry() *hf.Client {
|
||||
}
|
||||
|
||||
func (m consoleModel) fetchHF() tea.Cmd {
|
||||
c, q, sort := m.registry(), m.hf.query, m.hf.sort
|
||||
c, q, sort, st := m.registry(), m.hf.query, m.hf.sort, m.hf.safetensors
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
defer cancel()
|
||||
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50}
|
||||
hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: !st, Limit: 50}
|
||||
if st {
|
||||
hq.Filters = []string{"safetensors"}
|
||||
}
|
||||
// Browsing: models that chat. A search finds the rest too, an
|
||||
// embedding model included.
|
||||
if q == "" {
|
||||
hq.Task = "text-generation"
|
||||
}
|
||||
got, err := c.Explore(ctx, hq)
|
||||
return consoleHFMsg{query: q, sort: sort, models: got, err: err}
|
||||
return consoleHFMsg{query: q, sort: sort, st: st, models: got, err: err}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +93,17 @@ func (m *consoleModel) fetchRepo(id string) tea.Cmd {
|
||||
// repoMenu offers a repository's GGUF files, smallest first as listed.
|
||||
func (m consoleModel) repoMenu(r hf.Repo) *menuDialog {
|
||||
var items []menuItem
|
||||
if m.hf.safetensors {
|
||||
title := r.ID
|
||||
set, err := modelserve.SafetensorsSet(r)
|
||||
if err != nil {
|
||||
return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}}
|
||||
}
|
||||
return &menuDialog{title: title, items: []menuItem{{
|
||||
fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)),
|
||||
func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil },
|
||||
}}}
|
||||
}
|
||||
for _, set := range hf.Group(r.Files) {
|
||||
if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") {
|
||||
continue
|
||||
@@ -124,13 +143,19 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
|
||||
note += ", licence " + r.License
|
||||
}
|
||||
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes."
|
||||
fields := []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "4096", "what one request may use"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
}
|
||||
// A vision model: its projector, loaded beside it, is what reads images.
|
||||
if proj := modelserve.Projectors(r); len(proj) > 0 {
|
||||
fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0]))
|
||||
fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model"
|
||||
}
|
||||
m.openForm(&formDialog{title: "Serve " + r.ID, note: note,
|
||||
fields: []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "4096", "what one request may use"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
},
|
||||
fields: fields,
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 0 {
|
||||
@@ -147,6 +172,50 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
|
||||
if v[3] != "" {
|
||||
args = append(args, "--description="+v[3])
|
||||
}
|
||||
if len(v) > 4 && !strings.HasPrefix(v[4], "none") {
|
||||
args = append(args, "--projector="+v[4])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) {
|
||||
server := "--server=" + m.client.base
|
||||
var classes []string
|
||||
for _, g := range m.data.sum.GPUs {
|
||||
classes = append(classes, g.Class)
|
||||
}
|
||||
if len(classes) == 0 {
|
||||
m.dialog, m.dialogTitle = "error", "No GPUs"
|
||||
m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" +
|
||||
"F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key")
|
||||
return
|
||||
}
|
||||
m.openForm(&formDialog{title: "Serve " + r.ID,
|
||||
note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+
|
||||
"Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)),
|
||||
fields: []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "8192", "what one request may use; vLLM's --max-model-len"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
choiceF("GPU class", classes, classes[0]),
|
||||
textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} {
|
||||
if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) {
|
||||
return nil, fmt.Errorf("%s is a whole number", what)
|
||||
}
|
||||
}
|
||||
args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4],
|
||||
"--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
|
||||
if v[1] == "0" {
|
||||
args = append(args, "--on-demand")
|
||||
}
|
||||
if v[3] != "" {
|
||||
args = append(args, "--description="+v[3])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
}
|
||||
@@ -164,7 +233,7 @@ func (m *consoleModel) hfSearchDialog() {
|
||||
func (m consoleModel) hfPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave"
|
||||
keys = "↑↓ row · Enter choose · / search · S order · F GGUF or original weights · ← back · Q leave"
|
||||
}
|
||||
what := hfSorts[m.hf.sort].label
|
||||
if m.hf.query != "" {
|
||||
@@ -173,7 +242,11 @@ func (m consoleModel) hfPanel(pw int) (string, string, string) {
|
||||
if m.hf.query == "" {
|
||||
what = "text generation, " + what
|
||||
}
|
||||
head := "\n GGUF models on Hugging Face: " + what + "\n\n"
|
||||
kind := "GGUF models, for llama.cpp"
|
||||
if m.hf.safetensors {
|
||||
kind = "Original weights, for vLLM on GPUs"
|
||||
}
|
||||
head := "\n " + kind + " on Hugging Face: " + what + "\n\n"
|
||||
switch {
|
||||
case m.hf.err != nil:
|
||||
return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) +
|
||||
|
||||
+82
-25
@@ -62,26 +62,75 @@ func (m *consoleModel) mirrorCall(title string, do func(context.Context, *mirror
|
||||
}
|
||||
}
|
||||
|
||||
// usedBy names the catalog's models whose source is this file.
|
||||
func (m consoleModel) usedBy(base string, s mirror.Status) []string {
|
||||
u := mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(base)
|
||||
dir := strings.TrimSuffix(mirror.Entry{Model: s.Model, Version: s.Version}.URL(base), "/") + "/"
|
||||
// mirrorGroup is one version of one model on the mirror: a file, or the set
|
||||
// of files a model of several is fetched as.
|
||||
type mirrorGroup struct {
|
||||
model, version string
|
||||
files []mirror.Status
|
||||
}
|
||||
|
||||
func (g mirrorGroup) bytes() (done, total int64) {
|
||||
for _, f := range g.files {
|
||||
done += f.BytesDone
|
||||
total += max(f.BytesTotal, f.BytesDone)
|
||||
}
|
||||
return done, total
|
||||
}
|
||||
|
||||
// source is what a model's definition names for it: the file, or the
|
||||
// directory of a set.
|
||||
func (g mirrorGroup) source(base string) string {
|
||||
if len(g.files) == 1 {
|
||||
return mirror.Entry{Model: g.model, Version: g.version, File: g.files[0].File}.URL(base)
|
||||
}
|
||||
return strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorGroups() []mirrorGroup {
|
||||
var out []mirrorGroup
|
||||
at := map[string]int{}
|
||||
for _, st := range m.mirror.res.Models {
|
||||
k := st.Model + "/" + st.Version
|
||||
i, ok := at[k]
|
||||
if !ok {
|
||||
i = len(out)
|
||||
at[k] = i
|
||||
out = append(out, mirrorGroup{model: st.Model, version: st.Version})
|
||||
}
|
||||
out[i].files = append(out[i].files, st)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// usedBy names the catalog's models that run from a version's files.
|
||||
func (m consoleModel) usedBy(base string, g mirrorGroup) []string {
|
||||
dir := strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
|
||||
var names []string
|
||||
for _, md := range m.data.models {
|
||||
if md.Source == u || strings.HasPrefix(md.Source, dir) {
|
||||
if strings.HasPrefix(md.Source, dir) {
|
||||
names = append(names, md.Name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
|
||||
what := s.Model + "/" + s.Version
|
||||
used := m.usedBy(m.mirror.res.Base, s)
|
||||
items := []menuItem{{"Where nodes fetch it", func(m *consoleModel) tea.Cmd {
|
||||
func (m consoleModel) mirrorMenu(g mirrorGroup) *menuDialog {
|
||||
what := g.model + "/" + g.version
|
||||
used := m.usedBy(m.mirror.res.Base, g)
|
||||
items := []menuItem{{"Its files, and where nodes fetch them", func(m *consoleModel) tea.Cmd {
|
||||
var b strings.Builder
|
||||
for _, f := range g.files {
|
||||
sum := ""
|
||||
if f.SHA256 != "" {
|
||||
sum = sDim.Render(" sha256 " + f.SHA256[:min(12, len(f.SHA256))])
|
||||
}
|
||||
fmt.Fprintf(&b, "%-50s %9s%s\n", truncate(f.File, 50), bytesShort(max(f.BytesTotal, f.BytesDone)), sum)
|
||||
if f.Source != "" {
|
||||
b.WriteString(sDim.Render(" from "+truncate(f.Source, 90)) + "\n")
|
||||
}
|
||||
}
|
||||
m.dialog, m.dialogTitle = "error", what
|
||||
m.dialogText = mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(m.mirror.res.Base) +
|
||||
"\n\n" + sDim.Render("the source to give a model definition · any key")
|
||||
m.dialogText = b.String() + "\nA model's source: " + g.source(m.mirror.res.Base) + "\n\n" + sDim.Render("any key")
|
||||
return nil
|
||||
}}}
|
||||
items = append(items, menuItem{"Remove from the mirror…", func(m *consoleModel) tea.Cmd {
|
||||
@@ -93,7 +142,7 @@ func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
|
||||
m.confirmD = &confirmDialog{title: "Remove " + what, text: text,
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.mirrorCall("Remove "+what, func(ctx context.Context, c *mirror.Client) (string, error) {
|
||||
removed, err := c.Remove(ctx, s.Model, s.Version)
|
||||
removed, err := c.Remove(ctx, g.model, g.version)
|
||||
return fmt.Sprintf("Removed %d file(s).", len(removed)), err
|
||||
})
|
||||
}}
|
||||
@@ -139,24 +188,32 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
}
|
||||
head := "\n " + sDim.Render("Nodes fetch from "+m.mirror.res.Base) + "\n\n"
|
||||
var rows [][]string
|
||||
for _, s := range m.mirror.res.Models {
|
||||
state := string(s.State)
|
||||
switch s.State {
|
||||
case mirror.Fetching:
|
||||
if p, ok := s.Percent(); ok {
|
||||
state = sWarn.Render(fmt.Sprintf("fetching %d%%", p))
|
||||
} else {
|
||||
state = sWarn.Render("fetching " + bytesShort(s.BytesDone))
|
||||
for _, g := range m.mirrorGroups() {
|
||||
done, total := g.bytes()
|
||||
state := "present"
|
||||
for _, f := range g.files {
|
||||
switch f.State {
|
||||
case mirror.Fetching:
|
||||
state = sWarn.Render("fetching")
|
||||
if total > 0 {
|
||||
state = sWarn.Render(fmt.Sprintf("fetching %d%%", done*100/total))
|
||||
}
|
||||
case mirror.Failed:
|
||||
state = sBad.Render("failed")
|
||||
}
|
||||
if f.State != mirror.Present {
|
||||
break
|
||||
}
|
||||
case mirror.Failed:
|
||||
state = sBad.Render("failed")
|
||||
}
|
||||
size := bytesShort(s.BytesDone)
|
||||
used := strings.Join(m.usedBy(m.mirror.res.Base, s), ", ")
|
||||
what := g.files[0].File
|
||||
if len(g.files) > 1 {
|
||||
what = fmt.Sprintf("%d files", len(g.files))
|
||||
}
|
||||
used := strings.Join(m.usedBy(m.mirror.res.Base, g), ", ")
|
||||
if used == "" {
|
||||
used = sDim.Render("unused")
|
||||
}
|
||||
rows = append(rows, []string{s.Model, s.Version, s.File, state, size, used})
|
||||
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
|
||||
|
||||
Reference in new issue
Block a user