diff --git a/CHANGELOG.md b/CHANGELOG.md index 5509bd39..e8d1575b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,25 @@ is not an update. A model from `server.yaml` is reported but not changed: its source is edited there. +### Models of several files: vision projectors, original weights + +- A vision model's projector (`mmproj`) is fetched beside its GGUF weights + and loaded with them: `llmgw ctl model serve REPO --projector FILE`, or the + choice offered by the console. The model is marked as taking images. Nodes + load it once their agent is this version or later. +- `--format safetensors` serves a repository's original weights, with their + config.json, tokenizer and shard index, on vLLM across `--tp` GPUs of + `--gpu-class` (the fleet's one class when there is one). In the console, + **F** on Serve a model switches to them. +- The mirror keeps where it fetched each file beside it, and reports that + and the checksum after it restarts. The console's Mirror view shows one + row per version of a model, with its files on Enter. +- An update watches every file of a model, the projector and the tokenizer + included; one whose files the mirror cannot trace is found on the + registry by name, commit and size, or given with `llmgw ctl model origin`. +- `llmgw_model_update_available` is published, and `ModelUpdateAvailable` + alerts on it. + ### The manual, in the binary `llmgw docs` reads every document — installation, operation, the diff --git a/docs/console.md b/docs/console.md index 676e27b6..978db9c9 100644 --- a/docs/console.md +++ b/docs/console.md @@ -111,9 +111,13 @@ which): requests, tokens, errors and spend. team or project is guaranteed and may at most hold, and what it holds now. **A** adds one; on a row, change or remove it. -**Mirror** is what the model mirror holds, how far each download has got, and -which models use each file. **A** downloads a URL into it; on a row, where -nodes fetch it from, or remove it — with a warning when a model still uses it. +**Mirror** is what the model mirror holds, one row for each version of each +model — a file, or the set of files a model of several is: shards, a vision +projector, a config and tokenizer — with how far its download has got and +which models use it. **A** downloads a URL into it; on a row, its files with +their checksums and where each was fetched from, the source a model +definition names for it, or remove it — with a warning when a model still +uses it. **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. @@ -181,12 +185,18 @@ to restart it. Another control plane keeps its own copy. certificate and starts its agent (`sudo llmgw node add`). - **Replicate the store** — moves the store onto two more machines under Patroni (offered only while it is not replicated). -- **Serve a model** — Hugging Face's GGUF models: trending, most downloaded, - most liked or newest (**S** changes which), or **/** to search. Enter on - one lists its files with their sizes; the one chosen is fetched into the - mirror, checked, added to the catalog and started, and a replica is asked - to say hello (`llmgw ctl model serve`). `mirror.registry` points it at - another registry, for a site without the internet. +- **Serve a model** — Hugging Face's models: trending, most downloaded, + most liked or newest (**S** changes which), or **/** to search. **F** + switches between GGUF files, which llama.cpp runs on the CPU or a GPU, and + original safetensors weights, which vLLM runs on GPUs. Enter on a GGUF + repository lists its files with their sizes; a vision model offers its + projector too, which is fetched beside it and loaded with it so the model + reads images. Enter on original weights offers them with their config and + tokenizer, a GPU class and how many GPUs a replica spans. What is chosen is + fetched into the mirror, checked, added to the catalog and started, and a + replica is asked to say hello (`llmgw ctl model serve`). + `mirror.registry` points it at another registry, for a site without the + internet. - **Upgrade the cluster** to a release: this machine, then every node in turn (`sudo llmgw ctl upgrade`). - **Restart all replicas**, one at a time. diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index ad351286..33cadf47 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -93,7 +93,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { bin := l.bin() p := s.Profile - model, err := ggufIn(p.ModelPath) + model, projector, err := ggufIn(p.ModelPath) if err != nil { return nil, err } @@ -120,6 +120,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { if p.ServedModelName != "" { args = append(args, "--alias", p.ServedModelName) } + // A vision model's projector, fetched beside it: what turns an image in + // a request into something the language model reads. + if projector != "" && !hasFlag(p.Args, "--mmproj", "-mm") { + args = append(args, "--mmproj", projector) + } if !hasFlag(p.Args, "--parallel", "-np") { n := l.cfg.Parallel if n <= 0 { @@ -203,30 +208,46 @@ func hasFlag(args []string, names ...string) bool { // // A path that is already a file is returned untouched, which is every // single-file model and every hand-staged one. -func ggufIn(path string) (string, error) { +// +// A vision projector in the directory — an mmproj file — is not the model: +// it is returned second, to be loaded beside it. +func ggufIn(path string) (model, projector string, err error) { st, err := os.Stat(path) if err != nil || !st.IsDir() { // Not a directory, or not there at all — the second is the engine's to // report, with the message it would have given anyway. - return path, nil //nolint:nilerr // a missing path is the engine's error to give + return path, "", nil //nolint:nilerr // a missing path is the engine's error to give } entries, err := os.ReadDir(path) if err != nil { - return "", err + return "", "", err } - var names []string + var names, projectors []string for _, e := range entries { - if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { + lower := strings.ToLower(e.Name()) + if e.IsDir() || !strings.HasSuffix(lower, ".gguf") { + continue + } + if strings.Contains(lower, "mmproj") { + projectors = append(projectors, e.Name()) + } else { names = append(names, e.Name()) } } if len(names) == 0 { - return "", fmt.Errorf("%s holds no .gguf file", path) + if len(projectors) > 0 { + return "", "", fmt.Errorf("%s holds a vision projector and no model", path) + } + return "", "", fmt.Errorf("%s holds no .gguf file", path) } // Sorted, so a split model's first part comes first: the parts are named // "-00001-of-00004" and lexical order is shard order. sort.Strings(names) - return filepath.Join(path, names[0]), nil + if len(projectors) > 0 { + sort.Strings(projectors) + projector = filepath.Join(path, projectors[0]) + } + return filepath.Join(path, names[0]), projector, nil } // UseInstalled points the engine at a build the agent installed and checked. diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 07ec11b5..0e59cdfb 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -204,7 +204,7 @@ func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) { } } - got, err := ggufIn(dir) + got, _, err := ggufIn(dir) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestAPlainFilePathIsUntouched(t *testing.T) { if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { t.Fatal(err) } - if got, _ := ggufIn(p); got != p { + if got, _, _ := ggufIn(p); got != p { t.Errorf("expected the path unchanged, got %s", got) } } @@ -231,7 +231,7 @@ func TestADirectoryWithNoWeightsSaysSo(t *testing.T) { if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil { t.Fatal(err) } - if _, err := ggufIn(dir); err == nil { + if _, _, err := ggufIn(dir); err == nil { t.Error("a directory with no .gguf in it is not a model llama.cpp can load") } } @@ -240,7 +240,7 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) { // The engine's own "no such file" names the path and the flag it came // from, which is more use than anything this could say. p := "/nonexistent/model.gguf" - got, err := ggufIn(p) + got, _, err := ggufIn(p) if err != nil || got != p { t.Errorf("expected it passed through, got %q %v", got, err) } @@ -361,3 +361,28 @@ func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) { t.Error("a one-GPU replica was given a split mode") } } + +func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) { + dir := t.TempDir() + // "mmproj" sorts before the model's name; it must still not be taken for it. + for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} { + if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}}) + if err != nil { + t.Fatal(err) + } + args := strings.Join(cmds[0].Args, " ") + if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) || + !strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) { + t.Errorf("%s", args) + } + // One the profile names itself is left as it is. + cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir, + Args: []string{"--mmproj", "/elsewhere.gguf"}}}) + if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 { + t.Errorf("two projectors: %v", cmds[0].Args) + } +} diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 8c795011..79e9c274 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" ) @@ -24,15 +25,27 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { "that the nodes' llama.cpp can load it and sizes its memory from its own\n" + "header, adds it to the catalog, waits for a replica, and asks it to say\n" + "hello. Each step is shown as it happens; the first that fails stops it.\n\n" + + "A vision model's projector (--projector) is fetched beside it and loaded with\n" + + "it. --format safetensors serves the original weights instead, with their\n" + + "config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" + " llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" + - " llmgw ctl model serve owner/big-GGUF --on-demand", + " llmgw ctl model serve owner/big-GGUF --on-demand\n" + + " llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" + + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { req.Repo = args[0] if onDemand { req.Replicas = 0 } + // One kind of GPU in the fleet: that one, unless told. + if req.Format == "safetensors" && req.GPUClass == "" { + var sum adminapi.Summary + if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 { + req.GPUClass = sum.GPUs[0].Class + } + } body, _ := json.Marshal(req) var job modelserve.Job if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { @@ -44,6 +57,10 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f := c.Flags() f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)") f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant") + f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it") + f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs") + f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)") + f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans") f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 4d59198a..94fbd089 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -292,7 +292,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case consoleMirrorMsg: m.mirror = msg case consoleHFMsg: - if msg.query == m.hf.query && msg.sort == m.hf.sort { + if msg.query == m.hf.query && msg.sort == m.hf.sort && msg.st == m.hf.safetensors { m.hf.models, m.hf.err = msg.models, msg.err if m.hf.models == nil && msg.err == nil { m.hf.models = []hf.Model{} @@ -447,7 +447,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cQuotas: m.menuD = m.quotaMenu(m.quotas[m.row]) case cMirror: - m.menuD = m.mirrorMenu(m.mirror.res.Models[m.row]) + m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: @@ -464,6 +464,12 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { if m.menu == cServe { m.hfSearchDialog() } + case "f": + if m.menu == cServe { + m.hf.safetensors = !m.hf.safetensors + m.hf.models, m.hf.err, m.row = nil, nil, 0 + return m, m.fetchHF() + } case "s": if m.menu == cServe { m.hf.sort = (m.hf.sort + 1) % len(hfSorts) @@ -824,7 +830,7 @@ func (m consoleModel) rowCount() int { case cQuotas: return len(m.quotas) case cMirror: - return len(m.mirror.res.Models) + return len(m.mirrorGroups()) case cServe: return len(m.hf.models) case cAccounts: diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 4b106bb4..18eab3fe 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -11,6 +11,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" ) @@ -24,15 +25,19 @@ var hfSorts = []struct { }{{"trending", hf.SortTrending}, {"most downloaded", hf.SortDownloads}, {"most liked", hf.SortLikes}, {"newest", hf.SortCreated}} type hfState struct { - query string - sort int - models []hf.Model - err error + query string + sort int + // safetensors lists original weights, for vLLM on GPUs, rather than + // GGUF files for llama.cpp. + safetensors bool + models []hf.Model + err error } type consoleHFMsg struct { query string sort int + st bool models []hf.Model err error } @@ -56,18 +61,21 @@ func (m consoleModel) registry() *hf.Client { } func (m consoleModel) fetchHF() tea.Cmd { - c, q, sort := m.registry(), m.hf.query, m.hf.sort + c, q, sort, st := m.registry(), m.hf.query, m.hf.sort, m.hf.safetensors return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) defer cancel() - hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: true, Limit: 50} + hq := hf.Query{Search: q, Sort: hfSorts[sort].sort, GGUF: !st, Limit: 50} + if st { + hq.Filters = []string{"safetensors"} + } // Browsing: models that chat. A search finds the rest too, an // embedding model included. if q == "" { hq.Task = "text-generation" } got, err := c.Explore(ctx, hq) - return consoleHFMsg{query: q, sort: sort, models: got, err: err} + return consoleHFMsg{query: q, sort: sort, st: st, models: got, err: err} } } @@ -85,6 +93,17 @@ func (m *consoleModel) fetchRepo(id string) tea.Cmd { // repoMenu offers a repository's GGUF files, smallest first as listed. func (m consoleModel) repoMenu(r hf.Repo) *menuDialog { var items []menuItem + if m.hf.safetensors { + title := r.ID + set, err := modelserve.SafetensorsSet(r) + if err != nil { + return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}} + } + return &menuDialog{title: title, items: []menuItem{{ + fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)), + func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil }, + }}} + } for _, set := range hf.Group(r.Files) { if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") { continue @@ -124,13 +143,19 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { note += ", licence " + r.License } note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes." + fields := []formField{ + textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("replicas", "1", "0 runs none until it is first asked for"), + textF("context length", "4096", "what one request may use"), + textF("description", "", "optional: shown to clients in /v1/models"), + } + // A vision model: its projector, loaded beside it, is what reads images. + if proj := modelserve.Projectors(r); len(proj) > 0 { + fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0])) + fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model" + } m.openForm(&formDialog{title: "Serve " + r.ID, note: note, - fields: []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), - textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "4096", "what one request may use"), - textF("description", "", "optional: shown to clients in /v1/models"), - }, + fields: fields, submit: func(m *consoleModel, v []string) (tea.Cmd, error) { n, err := strconv.Atoi(v[1]) if err != nil || n < 0 { @@ -147,6 +172,50 @@ func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { if v[3] != "" { args = append(args, "--description="+v[3]) } + if len(v) > 4 && !strings.HasPrefix(v[4], "none") { + args = append(args, "--projector="+v[4]) + } + return m.runAfter(args...), nil + }}) +} + +func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) { + server := "--server=" + m.client.base + var classes []string + for _, g := range m.data.sum.GPUs { + classes = append(classes, g.Class) + } + if len(classes) == 0 { + m.dialog, m.dialogTitle = "error", "No GPUs" + m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" + + "F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key") + return + } + m.openForm(&formDialog{title: "Serve " + r.ID, + note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+ + "Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)), + fields: []formField{ + textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("replicas", "1", "0 runs none until it is first asked for"), + textF("context length", "8192", "what one request may use; vLLM's --max-model-len"), + textF("description", "", "optional: shown to clients in /v1/models"), + choiceF("GPU class", classes, classes[0]), + textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} { + if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) { + return nil, fmt.Errorf("%s is a whole number", what) + } + } + args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4], + "--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} + if v[1] == "0" { + args = append(args, "--on-demand") + } + if v[3] != "" { + args = append(args, "--description="+v[3]) + } return m.runAfter(args...), nil }}) } @@ -164,7 +233,7 @@ func (m *consoleModel) hfSearchDialog() { func (m consoleModel) hfPanel(pw int) (string, string, string) { keys := "→ or Enter open · ↑↓ choose · Q leave" if m.inPanel { - keys = "↑↓ row · Enter choose a file · / search · S order · ← back · Q leave" + keys = "↑↓ row · Enter choose · / search · S order · F GGUF or original weights · ← back · Q leave" } what := hfSorts[m.hf.sort].label if m.hf.query != "" { @@ -173,7 +242,11 @@ func (m consoleModel) hfPanel(pw int) (string, string, string) { if m.hf.query == "" { what = "text generation, " + what } - head := "\n GGUF models on Hugging Face: " + what + "\n\n" + kind := "GGUF models, for llama.cpp" + if m.hf.safetensors { + kind = "Original weights, for vLLM on GPUs" + } + head := "\n " + kind + " on Hugging Face: " + what + "\n\n" switch { case m.hf.err != nil: return "Serve from Hugging Face", head + " " + sBad.Render(m.hf.err.Error()) + diff --git a/internal/cli/tui_mirror.go b/internal/cli/tui_mirror.go index bd2295ab..68d0f693 100644 --- a/internal/cli/tui_mirror.go +++ b/internal/cli/tui_mirror.go @@ -62,26 +62,75 @@ func (m *consoleModel) mirrorCall(title string, do func(context.Context, *mirror } } -// usedBy names the catalog's models whose source is this file. -func (m consoleModel) usedBy(base string, s mirror.Status) []string { - u := mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(base) - dir := strings.TrimSuffix(mirror.Entry{Model: s.Model, Version: s.Version}.URL(base), "/") + "/" +// mirrorGroup is one version of one model on the mirror: a file, or the set +// of files a model of several is fetched as. +type mirrorGroup struct { + model, version string + files []mirror.Status +} + +func (g mirrorGroup) bytes() (done, total int64) { + for _, f := range g.files { + done += f.BytesDone + total += max(f.BytesTotal, f.BytesDone) + } + return done, total +} + +// source is what a model's definition names for it: the file, or the +// directory of a set. +func (g mirrorGroup) source(base string) string { + if len(g.files) == 1 { + return mirror.Entry{Model: g.model, Version: g.version, File: g.files[0].File}.URL(base) + } + return strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/" +} + +func (m consoleModel) mirrorGroups() []mirrorGroup { + var out []mirrorGroup + at := map[string]int{} + for _, st := range m.mirror.res.Models { + k := st.Model + "/" + st.Version + i, ok := at[k] + if !ok { + i = len(out) + at[k] = i + out = append(out, mirrorGroup{model: st.Model, version: st.Version}) + } + out[i].files = append(out[i].files, st) + } + return out +} + +// usedBy names the catalog's models that run from a version's files. +func (m consoleModel) usedBy(base string, g mirrorGroup) []string { + dir := strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/" var names []string for _, md := range m.data.models { - if md.Source == u || strings.HasPrefix(md.Source, dir) { + if strings.HasPrefix(md.Source, dir) { names = append(names, md.Name) } } return names } -func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog { - what := s.Model + "/" + s.Version - used := m.usedBy(m.mirror.res.Base, s) - items := []menuItem{{"Where nodes fetch it", func(m *consoleModel) tea.Cmd { +func (m consoleModel) mirrorMenu(g mirrorGroup) *menuDialog { + what := g.model + "/" + g.version + used := m.usedBy(m.mirror.res.Base, g) + items := []menuItem{{"Its files, and where nodes fetch them", func(m *consoleModel) tea.Cmd { + var b strings.Builder + for _, f := range g.files { + sum := "" + if f.SHA256 != "" { + sum = sDim.Render(" sha256 " + f.SHA256[:min(12, len(f.SHA256))]) + } + fmt.Fprintf(&b, "%-50s %9s%s\n", truncate(f.File, 50), bytesShort(max(f.BytesTotal, f.BytesDone)), sum) + if f.Source != "" { + b.WriteString(sDim.Render(" from "+truncate(f.Source, 90)) + "\n") + } + } m.dialog, m.dialogTitle = "error", what - m.dialogText = mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(m.mirror.res.Base) + - "\n\n" + sDim.Render("the source to give a model definition · any key") + m.dialogText = b.String() + "\nA model's source: " + g.source(m.mirror.res.Base) + "\n\n" + sDim.Render("any key") return nil }}} items = append(items, menuItem{"Remove from the mirror…", func(m *consoleModel) tea.Cmd { @@ -93,7 +142,7 @@ func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog { m.confirmD = &confirmDialog{title: "Remove " + what, text: text, yes: func(m *consoleModel) tea.Cmd { return m.mirrorCall("Remove "+what, func(ctx context.Context, c *mirror.Client) (string, error) { - removed, err := c.Remove(ctx, s.Model, s.Version) + removed, err := c.Remove(ctx, g.model, g.version) return fmt.Sprintf("Removed %d file(s).", len(removed)), err }) }} @@ -139,24 +188,32 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) { } head := "\n " + sDim.Render("Nodes fetch from "+m.mirror.res.Base) + "\n\n" var rows [][]string - for _, s := range m.mirror.res.Models { - state := string(s.State) - switch s.State { - case mirror.Fetching: - if p, ok := s.Percent(); ok { - state = sWarn.Render(fmt.Sprintf("fetching %d%%", p)) - } else { - state = sWarn.Render("fetching " + bytesShort(s.BytesDone)) + for _, g := range m.mirrorGroups() { + done, total := g.bytes() + state := "present" + for _, f := range g.files { + switch f.State { + case mirror.Fetching: + state = sWarn.Render("fetching") + if total > 0 { + state = sWarn.Render(fmt.Sprintf("fetching %d%%", done*100/total)) + } + case mirror.Failed: + state = sBad.Render("failed") + } + if f.State != mirror.Present { + break } - case mirror.Failed: - state = sBad.Render("failed") } - size := bytesShort(s.BytesDone) - used := strings.Join(m.usedBy(m.mirror.res.Base, s), ", ") + what := g.files[0].File + if len(g.files) > 1 { + what = fmt.Sprintf("%d files", len(g.files)) + } + used := strings.Join(m.usedBy(m.mirror.res.Base, g), ", ") if used == "" { used = sDim.Render("unused") } - rows = append(rows, []string{s.Model, s.Version, s.File, state, size, used}) + rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used}) } if len(rows) == 0 { return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index d68491d3..8cda83b7 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -87,6 +87,15 @@ type Request struct { Quant string `json:"quant,omitempty"` // File picks it by name instead. File string `json:"file,omitempty"` + // Projector is a vision model's mmproj file in the repository, fetched + // beside the weights and loaded with them; empty serves text alone. + Projector string `json:"projector,omitempty"` + // Format is "gguf", run by llama.cpp, or "safetensors": the original + // weights with their config and tokenizer, run by vLLM on GPUs of + // GPUClass, across TP of them. + Format string `json:"format,omitempty"` + GPUClass string `json:"gpu_class,omitempty"` + TP int `json:"tp,omitempty"` // Name is the catalog name; the repository suggests one otherwise. Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` @@ -179,6 +188,20 @@ func (s *Server) Start(req Request) (Job, error) { if req.Replicas < 0 { return Job{}, errors.New("replicas cannot be negative") } + switch req.Format { + case "", "gguf": + req.Format = "gguf" + case "safetensors": + if req.GPUClass == "" { + return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it") + } + if req.Projector != "" { + return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files") + } + req.TP = max(req.TP, 1) + default: + return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) + } if req.Quant == "" && req.File == "" { req.Quant = "Q4_K_M" } @@ -251,11 +274,28 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo)) return } - set, err := pick(repo, req) + pickSet := pick + if req.Format == "safetensors" { + pickSet = pickSafetensors + } + set, err := pickSet(repo, req) if err != nil { s.fail(j, "registry", err) return } + if req.Projector != "" { + i := slices.IndexFunc(repo.Files, func(f hf.File) bool { + return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) && + strings.Contains(strings.ToLower(f.Path), "mmproj") + }) + if i < 0 { + s.fail(j, "registry", fmt.Errorf("%s has no vision projector %q; it has: %s", repo.ID, req.Projector, + strings.Join(Projectors(repo), ", "))) + return + } + set.Files = append(set.Files, repo.Files[i]) + set.Bytes += repo.Files[i].Bytes + } name := req.Name if name == "" { name = hf.ModelName(repo.ID) @@ -274,10 +314,20 @@ func (s *Server) run(j *Job) { } // 3. Whether the nodes' llama.cpp loads it, and how much memory it needs. + if req.Format == "safetensors" { + // vLLM reads the model's shape from its config.json, which came + // with the weights; the GPUs it needs are the class and width asked. + s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass)) + s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ + Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100, + Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)}, + }) + return + } s.step(j, "check", Running, "reading its header from the mirror") var header gguf.Header - if !strings.HasSuffix(source, "/") { - v := gguf.Check(ctx, source, s.deps.LlamaVersion) + if u := headerURL(source, set); u != "" { + v := gguf.Check(ctx, u, s.deps.LlamaVersion) if !v.Servable { s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason)) return @@ -302,16 +352,24 @@ func (s *Server) run(j *Job) { } } + s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ + Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, + Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, + }) +} + +// addAndStart puts the model in the catalog with its profile, waits for a +// replica and asks it to answer. +func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) { + req := j.Request // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, - Profiles: []catalog.Profile{{ - Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, - }}, - Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, - Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, + Profiles: []catalog.Profile{prof}, + Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, + Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, + Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } s.step(j, "catalog", Running, "adding "+name) if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { @@ -593,3 +651,72 @@ func orUnknown(arch string) string { } return arch } + +// Projectors are a repository's vision projectors: the mmproj files a vision +// model loads beside its weights. +func Projectors(repo hf.Repo) []string { + var out []string + for _, f := range repo.Files { + if strings.HasSuffix(strings.ToLower(f.Path), ".gguf") && strings.Contains(strings.ToLower(f.Path), "mmproj") { + out = append(out, f.Path) + } + } + return out +} + +// headerURL is the file whose header says what the model is: the source +// itself, or in a directory the first part of the weights, not a projector. +func headerURL(source string, set hf.FileSet) string { + if !strings.HasSuffix(source, "/") { + return source + } + var names []string + for _, f := range set.Files { + name := mirror.LastPathElement(f.Path) + lower := strings.ToLower(name) + if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") { + names = append(names, name) + } + } + if len(names) == 0 { + return "" + } + slices.Sort(names) + return source + names[0] +} + +// SafetensorsSet is what serving a repository's original weights fetches. +func SafetensorsSet(repo hf.Repo) (hf.FileSet, error) { return pickSafetensors(repo, Request{}) } + +// pickSafetensors chooses a repository's original weights and the files an +// engine reads beside them: config.json, the tokenizer, the shard index. +func pickSafetensors(repo hf.Repo, _ Request) (hf.FileSet, error) { + var weights []hf.File + for _, f := range repo.Files { + if !strings.Contains(f.Path, "/") && strings.HasSuffix(f.Path, ".safetensors") { + weights = append(weights, f) + } + } + sets := hf.Group(weights) + if len(sets) == 0 { + return hf.FileSet{}, fmt.Errorf("%s has no safetensors weights at its top", repo.ID) + } + // Sharded "model-00001-of-00004" or a single "model.safetensors"; a + // repository with a second, consolidated copy keeps it under another name. + set := sets[0] + for _, st := range sets { + if strings.HasPrefix(st.Name, "model") { + set = st + break + } + } + companions := hf.Companions(repo.Files) + if !slices.ContainsFunc(companions, func(f hf.File) bool { return f.Path == "config.json" }) { + return hf.FileSet{}, fmt.Errorf("%s has no config.json, which vLLM reads the model's shape from", repo.ID) + } + for _, f := range companions { + set.Files = append(set.Files, f) + set.Bytes += f.Bytes + } + return set, nil +} diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go index 7c4fdfe3..2760628a 100644 --- a/internal/control/modelserve/modelserve_test.go +++ b/internal/control/modelserve/modelserve_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -223,3 +224,68 @@ func TestARepositoryThatIsNotOneIsRefused(t *testing.T) { } } } + +func TestAVisionModelIsServedWithItsProjectorBesideIt(t *testing.T) { + files := []hf.File{ + {Path: "gemma-3-4b-it-Q4_K_M.gguf", Bytes: 2 << 30, SHA256: "aa"}, + {Path: "mmproj-gemma-3-4b-it-f16.gguf", Bytes: 800 << 20, SHA256: "bb"}, + } + s, m, c, _ := setup(t, files, map[string]string{"gemma-3-4b-it-Q4_K_M.gguf": "aa", "mmproj-gemma-3-4b-it-f16.gguf": "bb"}) + job, err := s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", + Projector: "mmproj-gemma-3-4b-it-f16.gguf", Replicas: 1}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + if len(m.got) != 2 { + t.Fatalf("fetched %+v", m.got) + } + got := c.added[0] + if !strings.HasSuffix(got.Source, "/") || !got.Capabilities.Vision || len(got.Origin.Files) != 2 { + t.Errorf("%s vision=%v origin=%+v", got.Source, got.Capabilities.Vision, got.Origin) + } + // A projector the repository does not have is named with those it has. + job, _ = s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", Projector: "mmproj-x.gguf", Name: "g2"}) + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "mmproj-gemma-3-4b-it-f16.gguf") { + t.Errorf("%+v", j) + } +} + +func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) { + files := []hf.File{ + {Path: "model-00001-of-00002.safetensors", Bytes: 4 << 30, SHA256: "a1"}, + {Path: "model-00002-of-00002.safetensors", Bytes: 3 << 30, SHA256: "a2"}, + {Path: "model.safetensors.index.json", Bytes: 30000}, + {Path: "config.json", Bytes: 700}, + {Path: "tokenizer.json", Bytes: 7 << 20}, + {Path: "tokenizer_config.json", Bytes: 7000}, + {Path: "README.md", Bytes: 9000}, + {Path: "original/consolidated.safetensors", Bytes: 7 << 30}, + } + s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"}) + if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil { + t.Error("safetensors with no GPU class was taken") + } + job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + var got []string + for _, r := range m.got { + got = append(got, r.File) + } + if len(got) != 6 || slices.Contains(got, "README.md") || slices.Contains(got, "consolidated.safetensors") { + t.Errorf("fetched %v", got) + } + mod := c.added[0] + p := mod.Profiles[0] + if !strings.HasSuffix(mod.Source, "/") || p.Engine != "vllm" || p.GPUClass != "l40s-48g" || p.TP != 1 || + !slices.Contains(p.Args, "--max-model-len") { + t.Errorf("%s %+v", mod.Source, p) + } +} diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index 62bebb64..c34615af 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -509,9 +509,9 @@ func (s *Server) runUpdate(j *Job) { // 3. Whether the nodes load it, and whether it still fits what is // reserved for it. s.step(j, "check", Running, "reading its header from the mirror") - detail := "a set of shards; not read" - if !strings.HasSuffix(source, "/") { - v := gguf.Check(ctx, source, s.deps.LlamaVersion) + detail := "no header to read" + if u := headerURL(source, set); u != "" { + v := gguf.Check(ctx, u, s.deps.LlamaVersion) if !v.Servable { s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) return diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 2c9edf4c..f67d548c 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -68,6 +68,10 @@ type Model struct { // GGUF is what the registry read from the repository's weights, absent // for one that has none. GGUF *GGUFInfo `json:"gguf"` + // Safetensors is the registry's summary of original weights. + Safetensors *struct { + Total int64 `json:"total"` + } `json:"safetensors,omitempty"` // Siblings are the repository's files, when asked for with WithFiles. Siblings []struct { RFilename string `json:"rfilename"` @@ -84,10 +88,13 @@ type GGUFInfo struct { // Params is the parameter count, 0 when the registry does not know it. func (m Model) Params() int64 { - if m.GGUF == nil { - return 0 + switch { + case m.GGUF != nil: + return m.GGUF.Total + case m.Safetensors != nil: + return m.Safetensors.Total } - return m.GGUF.Total + return 0 } // Architecture is the GGUF architecture, empty when unknown. @@ -172,7 +179,7 @@ type Query struct { // exploreFields is everything a listing shows. Asking for any field with // expand returns only the fields asked for, so all of them are named. var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified", - "createdAt", "gated", "private", "trendingScore", "gguf"} + "createdAt", "gated", "private", "trendingScore", "gguf", "safetensors"} // Explore lists repositories: the trending ones, the newest, or a search. //