Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
+16
-13
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
@@ -25,27 +24,26 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||
"Without --quant or --file, the file is the one that suits the cluster best,\n" +
|
||||
"and where it runs is the plan's best way for it: on one card, on several of\n" +
|
||||
"a node's, over several nodes on a fabric, or on a CPU node (`llmgw ctl model\n" +
|
||||
"plan REPO` lists them). --cpu or --gpu-class with --gpus and --nodes say it\n" +
|
||||
"instead; --fallbacks adds the next best ways for when the first is busy.\n\n" +
|
||||
"A vision model's projector (--projector) is fetched beside it and loaded with\n" +
|
||||
"it. --format safetensors serves the original weights instead, with their\n" +
|
||||
"config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" +
|
||||
"config.json and tokenizer, on vLLM.\n\n" +
|
||||
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand\n" +
|
||||
" llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g",
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --fallbacks\n" +
|
||||
" llmgw ctl model serve meta-llama/Llama-3.1-405B-Instruct --format safetensors --gpu-class h100-80g --gpus 8 --nodes 2",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Repo = args[0]
|
||||
if onDemand {
|
||||
req.Replicas = 0
|
||||
}
|
||||
// One kind of GPU in the fleet: that one, unless told.
|
||||
if req.Format == "safetensors" && req.GPUClass == "" {
|
||||
var sum adminapi.Summary
|
||||
if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 {
|
||||
req.GPUClass = sum.GPUs[0].Class
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
@@ -55,12 +53,17 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
|
||||
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)")
|
||||
f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans")
|
||||
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
|
||||
f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)")
|
||||
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
|
||||
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
|
||||
_ = f.MarkHidden("tp")
|
||||
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
|
||||
+102
-5
@@ -2,11 +2,15 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
@@ -26,13 +30,20 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
context int
|
||||
params int64
|
||||
precs []string
|
||||
format string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "plan <huggingface-repo>",
|
||||
Short: "Work out whether a model fits this cluster, and how it should be split",
|
||||
Long: "Reads the model's architecture from its repository and the hardware from\n" +
|
||||
"the running fleet, then prints every way it could run — fewest GPUs\n" +
|
||||
"first — and a catalog entry for the best one.\n\n" +
|
||||
Short: "Work out which of a model's files suits this cluster, and how it should be split",
|
||||
Long: "Lists every file the repository has — each GGUF quantisation, or with\n" +
|
||||
"--format safetensors its original weights — with the best way each could\n" +
|
||||
"run here: on one card, on several of a node's, over several nodes on a\n" +
|
||||
"fabric, or on a CPU node; what each card would hold and whether the cards\n" +
|
||||
"are free now. The model's shape is read from one file's header, nothing is\n" +
|
||||
"downloaded. `llmgw ctl model serve` without --quant takes the first.\n\n" +
|
||||
"With --precision it plans in the abstract instead, from the model's\n" +
|
||||
"config: every way it could run at bf16, fp8, q8_0 or q4_k_m, fewest GPUs\n" +
|
||||
"first, and a catalog entry for the best one.\n\n" +
|
||||
"Every candidate is listed rather than one answer, because the choice is\n" +
|
||||
"not the planner's to make: with cards to spare you want the\n" +
|
||||
"full-precision fit, and with one card free you want to know a quantised\n" +
|
||||
@@ -40,6 +51,11 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
repo := args[0]
|
||||
// The files the repository has, against the fleet as it is —
|
||||
// unless a precision or a size is given to plan in the abstract.
|
||||
if len(precs) == 0 && params == 0 {
|
||||
return planFiles(cmd, client, repo, format, context)
|
||||
}
|
||||
m, err := modelplan.FetchModel(cmd.Context(), repo)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -112,7 +128,9 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
|
||||
c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one")
|
||||
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
|
||||
c.Flags().StringSliceVar(&precs, "precision", nil, "plan in the abstract, for these precisions (bf16, fp8, q8_0, q4_k_m), "+
|
||||
"from the model's config rather than its files")
|
||||
c.Flags().StringVar(&format, "format", "gguf", "gguf files, for llama.cpp, or safetensors, the original weights for vLLM")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -238,3 +256,82 @@ func humanBytes(n int64) string {
|
||||
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
|
||||
}
|
||||
}
|
||||
|
||||
// planFiles prints the control plane's plan of a repository's files.
|
||||
func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string, context int) error {
|
||||
q := url.Values{"repo": {repo}, "format": {format}}
|
||||
if context > 0 {
|
||||
q.Set("context", strconv.Itoa(context))
|
||||
}
|
||||
var p modelserve.Plan
|
||||
if err := client().get(cmd.Context(), "/api/v1/plan?"+q.Encode(), &p); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s at %s", p.Repo, catalog.ShortRevision(p.Revision))
|
||||
if p.Arch != "" {
|
||||
fmt.Fprintf(out, ", %s", p.Arch)
|
||||
}
|
||||
if p.Params > 0 {
|
||||
fmt.Fprintf(out, ", %.1fB parameters", float64(p.Params)/1e9)
|
||||
}
|
||||
fmt.Fprintf(out, "; planned for a %d-token context\n", p.Context)
|
||||
if p.Gated {
|
||||
fmt.Fprintln(out, "gated: the mirror cannot fetch it without an account's approval")
|
||||
}
|
||||
if !p.Servable {
|
||||
fmt.Fprintf(out, "the nodes' engine cannot run it: %s\n", p.Reason)
|
||||
return errDoesNotFit
|
||||
}
|
||||
if p.Note != "" {
|
||||
fmt.Fprintln(out, "note: "+p.Note)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW")
|
||||
for _, a := range p.Artifacts {
|
||||
name := a.Name
|
||||
if a.Shards > 1 && a.Format == "gguf" {
|
||||
name += fmt.Sprintf(" (%d parts)", a.Shards)
|
||||
}
|
||||
if a.Best == nil {
|
||||
fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes))
|
||||
continue
|
||||
}
|
||||
b := a.Best
|
||||
now := "busy"
|
||||
if b.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
holds := fmt.Sprintf("%.1f GB a card", b.NeedGB)
|
||||
if b.CPU {
|
||||
holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB)
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
rec, ok := p.Recommended()
|
||||
if !ok {
|
||||
fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.")
|
||||
return errDoesNotFit
|
||||
}
|
||||
fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label())
|
||||
if len(p.Options) > 1 {
|
||||
fmt.Fprintln(out, "\nEvery way, best first:")
|
||||
w = tabTo(out)
|
||||
for _, o := range p.Options {
|
||||
now := "busy"
|
||||
if o.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
}
|
||||
fmt.Fprintf(out, "\nServe it: llmgw ctl model serve %s", p.Repo)
|
||||
if p.Format == "safetensors" {
|
||||
fmt.Fprint(out, " --format safetensors")
|
||||
}
|
||||
fmt.Fprintln(out, " --fallbacks")
|
||||
return nil
|
||||
}
|
||||
@@ -334,6 +334,7 @@ func newServerCmd() *cobra.Command {
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart,
|
||||
Pool: adminAPI.PoolNodes,
|
||||
})
|
||||
adminAPI.SetServe(serve)
|
||||
updateMetrics.Set(serve)
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -283,7 +285,41 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
|
||||
if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := newConsoleModel("127.0.0.1:9090", cfgPath)
|
||||
// The control plane's plan of the repository's files.
|
||||
plan := modelserve.Plan{Repo: "bartowski/Qwen2.5-3B-Instruct-GGUF", Revision: "abc", Format: "gguf", Context: 16384,
|
||||
Servable: true, Artifacts: []modelserve.PlannedArtifact{
|
||||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q4_K_M.gguf", Format: "gguf", Bytes: 2 << 30}},
|
||||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q8_0.gguf", Format: "gguf", Bytes: 3 << 30}},
|
||||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-F16.gguf", Format: "gguf", Bytes: 6 << 30}},
|
||||
}}
|
||||
gpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1,
|
||||
NeedGB: 5, HaveGB: 21.6, Hosts: 1, FreeNow: true}
|
||||
cpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 5, HaveGB: 27, Hosts: 2, FreeNow: true}
|
||||
for _, o := range []struct {
|
||||
art string
|
||||
o modelplan.Option
|
||||
}{{"Qwen2.5-3B-Instruct-Q8_0.gguf", gpu}, {"Qwen2.5-3B-Instruct-Q4_K_M.gguf", gpu},
|
||||
{"Qwen2.5-3B-Instruct-Q8_0.gguf", cpu}, {"Qwen2.5-3B-Instruct-F16.gguf", cpu}} {
|
||||
o.o.Artifact = o.art
|
||||
plan.Options = append(plan.Options, o.o)
|
||||
}
|
||||
for i := range plan.Artifacts {
|
||||
for j := range plan.Options {
|
||||
if plan.Options[j].Artifact == plan.Artifacts[i].Name {
|
||||
plan.Artifacts[i].Best = &plan.Options[j]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo {
|
||||
_ = json.NewEncoder(w).Encode(plan)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("[]"))
|
||||
}))
|
||||
defer admin.Close()
|
||||
m := newConsoleModel(admin.URL, cfgPath)
|
||||
m.loaded, m.menu, m.inPanel = true, cServe, true
|
||||
var tm tea.Model = m
|
||||
|
||||
@@ -303,19 +339,22 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
|
||||
tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
tm = run(tm, cmd)
|
||||
d := tm.(consoleModel).menuD
|
||||
if d == nil || len(d.items) != 2 {
|
||||
if d == nil || len(d.items) != 3 || !strings.Contains(d.items[0].label, "★") ||
|
||||
!strings.Contains(d.items[0].label, "Q8_0") || !strings.Contains(d.items[0].label, "1 × 6000-24g") ||
|
||||
!strings.Contains(d.items[2].label, "CPU") {
|
||||
t.Fatalf("files offered: %+v", d)
|
||||
}
|
||||
tm = press(tm, "down", "enter")
|
||||
tm = press(tm, "enter")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" {
|
||||
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" || len(f.fields[4].options) != 2 {
|
||||
t.Fatalf("form: %+v", f)
|
||||
}
|
||||
f.focus = len(f.fields)
|
||||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
act := tm.(consoleModel).action
|
||||
got := strings.Join(act.targets, " ")
|
||||
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") {
|
||||
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") ||
|
||||
!strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") {
|
||||
t.Fatalf("would run %q %q", act.kind, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +304,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.dialog, m.dialogTitle = "error", "Hugging Face"
|
||||
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
|
||||
} else {
|
||||
m.menuD = m.repoMenu(msg.repo)
|
||||
m.menuD = m.repoMenu(msg.plan)
|
||||
}
|
||||
case consolePeopleMsg:
|
||||
m.people = peopleData(msg)
|
||||
|
||||
+130
-98
@@ -3,6 +3,8 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// Serving from Hugging Face: the registry's GGUF models, trending or
|
||||
@@ -43,7 +46,7 @@ type consoleHFMsg struct {
|
||||
}
|
||||
|
||||
type consoleRepoMsg struct {
|
||||
repo hf.Repo
|
||||
plan modelserve.Plan
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -79,142 +82,171 @@ func (m consoleModel) fetchHF() tea.Cmd {
|
||||
}
|
||||
}
|
||||
|
||||
// fetchRepo asks the control plane for a repository's plan: its files, and
|
||||
// how each could run on this cluster.
|
||||
func (m *consoleModel) fetchRepo(id string) tea.Cmd {
|
||||
m.busy = "Reading " + id
|
||||
c := m.registry()
|
||||
m.busy = "Planning " + id
|
||||
c := m.client
|
||||
format := "gguf"
|
||||
if m.hf.safetensors {
|
||||
format = "safetensors"
|
||||
}
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
r, err := c.Repo(ctx, id)
|
||||
return consoleRepoMsg{r, err}
|
||||
var p modelserve.Plan
|
||||
err := c.get(ctx, "/api/v1/plan?"+url.Values{"repo": {id}, "format": {format}}.Encode(), &p)
|
||||
return consoleRepoMsg{p, err}
|
||||
}
|
||||
}
|
||||
|
||||
// repoMenu offers a repository's GGUF files, smallest first as listed.
|
||||
func (m consoleModel) repoMenu(r hf.Repo) *menuDialog {
|
||||
var items []menuItem
|
||||
if m.hf.safetensors {
|
||||
title := r.ID
|
||||
set, err := modelserve.SafetensorsSet(r)
|
||||
if err != nil {
|
||||
return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}}
|
||||
}
|
||||
return &menuDialog{title: title, items: []menuItem{{
|
||||
fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)),
|
||||
func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil },
|
||||
}}}
|
||||
// optionLabel is one way to run a file, with what it needs and whether the
|
||||
// hardware is free.
|
||||
func optionLabel(o modelplan.Option) string {
|
||||
now := sDim.Render("busy now")
|
||||
if o.FreeNow {
|
||||
now = sOK.Render("free")
|
||||
}
|
||||
for _, set := range hf.Group(r.Files) {
|
||||
if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") {
|
||||
continue
|
||||
need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB)
|
||||
if o.CPU {
|
||||
need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB)
|
||||
}
|
||||
return fmt.Sprintf("%s · %s · %s", o.Label(), need, now)
|
||||
}
|
||||
|
||||
// repoMenu offers a repository's files, each with the best way it runs here;
|
||||
// the plan's choice first.
|
||||
func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog {
|
||||
title := p.Repo
|
||||
if p.Gated {
|
||||
title += " · gated"
|
||||
}
|
||||
var items []menuItem
|
||||
if !p.Servable {
|
||||
return &menuDialog{title: title, items: []menuItem{{"The nodes' engine cannot run it: " + p.Reason,
|
||||
func(*consoleModel) tea.Cmd { return nil }}}}
|
||||
}
|
||||
rec, hasRec := p.Recommended()
|
||||
arts := append([]modelserve.PlannedArtifact{}, p.Artifacts...)
|
||||
// The recommended file first, then the rest as the repository lists them.
|
||||
for i, a := range arts {
|
||||
if hasRec && a.Name == rec.Artifact && i > 0 {
|
||||
arts = append([]modelserve.PlannedArtifact{a}, append(arts[:i:i], arts[i+1:]...)...)
|
||||
break
|
||||
}
|
||||
low := strings.ToLower(set.Name)
|
||||
// Projectors and imatrix data are not a model to serve on their own.
|
||||
if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") {
|
||||
continue
|
||||
}
|
||||
for _, a := range arts {
|
||||
name := a.Name
|
||||
if a.Format == "safetensors" {
|
||||
name = "original weights"
|
||||
}
|
||||
label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes))
|
||||
if set.Shards > 1 {
|
||||
label += fmt.Sprintf(" %d parts", set.Shards)
|
||||
label := fmt.Sprintf("%-40s %8s ", truncate(name, 40), bytesShort(a.Bytes))
|
||||
switch {
|
||||
case a.Best == nil:
|
||||
label += sBad.Render("fits nowhere")
|
||||
default:
|
||||
label += optionLabel(*a.Best)
|
||||
}
|
||||
set := set
|
||||
if hasRec && a.Name == rec.Artifact {
|
||||
label = sOK.Render("★ ") + label
|
||||
} else {
|
||||
label = " " + label
|
||||
}
|
||||
a := a
|
||||
items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd {
|
||||
m.serveForm(r, set)
|
||||
if a.Best == nil {
|
||||
m.dialog, m.dialogTitle = "error", a.Name
|
||||
m.dialogText = fmt.Sprintf("No card holds %s with a %d-token context, nor does any node's memory.\n"+
|
||||
"A smaller quantisation, or more cards free.\n\n%s", a.Name, p.Context, sDim.Render("any key"))
|
||||
return nil
|
||||
}
|
||||
m.serveForm(p, a)
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
title := r.ID
|
||||
if len(items) == 0 {
|
||||
items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}}
|
||||
}
|
||||
if r.Gated.Is() {
|
||||
title += " · gated"
|
||||
}
|
||||
return &menuDialog{title: title, items: items}
|
||||
}
|
||||
|
||||
func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) {
|
||||
func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) {
|
||||
server := "--server=" + m.client.base
|
||||
note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes))
|
||||
if r.Architecture != "" {
|
||||
note += ", " + r.Architecture
|
||||
var opts []modelplan.Option
|
||||
for _, o := range p.Options {
|
||||
if o.Artifact == a.Name {
|
||||
opts = append(opts, o)
|
||||
}
|
||||
}
|
||||
if r.License != "" {
|
||||
note += ", licence " + r.License
|
||||
var where []string
|
||||
for _, o := range opts {
|
||||
where = append(where, optionLabel(o))
|
||||
}
|
||||
note := fmt.Sprintf("%s, %s", a.Name, bytesShort(a.Bytes))
|
||||
if a.Format == "safetensors" {
|
||||
note = fmt.Sprintf("Original weights, %s in %d files with their config and tokenizer, for vLLM", bytesShort(a.Bytes), a.Shards)
|
||||
}
|
||||
if p.Arch != "" {
|
||||
note += ", " + p.Arch
|
||||
}
|
||||
if p.License != "" {
|
||||
note += ", licence " + p.License
|
||||
}
|
||||
if p.Note != "" {
|
||||
note += "\n" + sWarn.Render("! ") + p.Note
|
||||
}
|
||||
note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+
|
||||
"catalog, and a replica is started and asked to say hello.", p.Context)
|
||||
ctxDefault := "4096"
|
||||
if a.Format == "safetensors" {
|
||||
ctxDefault = strconv.Itoa(p.Context)
|
||||
}
|
||||
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes."
|
||||
fields := []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("name", hf.ModelName(p.Repo), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "4096", "what one request may use"),
|
||||
textF("context length", ctxDefault, "what one request may use"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
choiceF("runs on", where, where[0]),
|
||||
choiceF("when that is busy", []string{"the next best way", "wait for it"}, "the next best way"),
|
||||
}
|
||||
// A vision model: its projector, loaded beside it, is what reads images.
|
||||
if proj := modelserve.Projectors(r); len(proj) > 0 {
|
||||
fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0]))
|
||||
fields[4].hint = "every way this file fits, best first"
|
||||
proj := -1
|
||||
if a.Format == "gguf" && len(p.Projectors) > 0 {
|
||||
fields = append(fields, choiceF("images", append(append([]string{}, p.Projectors...), "none: text only"), p.Projectors[0]))
|
||||
fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model"
|
||||
proj = len(fields) - 1
|
||||
}
|
||||
m.openForm(&formDialog{title: "Serve " + r.ID, note: note,
|
||||
fields: fields,
|
||||
m.openForm(&formDialog{title: "Serve " + p.Repo, note: note, fields: fields,
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 0 {
|
||||
return nil, fmt.Errorf("replicas is a whole number")
|
||||
}
|
||||
if _, err := strconv.Atoi(v[2]); err != nil {
|
||||
if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 {
|
||||
return nil, fmt.Errorf("the context length is a whole number")
|
||||
}
|
||||
args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name,
|
||||
"--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
|
||||
args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1],
|
||||
"--context-length=" + v[2]}
|
||||
if a.Format == "safetensors" {
|
||||
args = append(args, "--format=safetensors")
|
||||
} else {
|
||||
args = append(args, "--file="+a.Name)
|
||||
}
|
||||
o := opts[slices.Index(where, v[4])]
|
||||
if o.CPU {
|
||||
args = append(args, "--cpu")
|
||||
} else {
|
||||
args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode),
|
||||
"--nodes="+strconv.Itoa(max(o.Nodes, 1)))
|
||||
}
|
||||
if strings.HasPrefix(v[5], "the next") && len(opts) > 1 {
|
||||
args = append(args, "--fallbacks")
|
||||
}
|
||||
if n == 0 {
|
||||
args = append(args, "--on-demand")
|
||||
}
|
||||
if v[3] != "" {
|
||||
args = append(args, "--description="+v[3])
|
||||
}
|
||||
if len(v) > 4 && !strings.HasPrefix(v[4], "none") {
|
||||
args = append(args, "--projector="+v[4])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) {
|
||||
server := "--server=" + m.client.base
|
||||
var classes []string
|
||||
for _, g := range m.data.sum.GPUs {
|
||||
classes = append(classes, g.Class)
|
||||
}
|
||||
if len(classes) == 0 {
|
||||
m.dialog, m.dialogTitle = "error", "No GPUs"
|
||||
m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" +
|
||||
"F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key")
|
||||
return
|
||||
}
|
||||
m.openForm(&formDialog{title: "Serve " + r.ID,
|
||||
note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+
|
||||
"Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)),
|
||||
fields: []formField{
|
||||
textF("name", hf.ModelName(r.ID), "what clients ask for"),
|
||||
textF("replicas", "1", "0 runs none until it is first asked for"),
|
||||
textF("context length", "8192", "what one request may use; vLLM's --max-model-len"),
|
||||
textF("description", "", "optional: shown to clients in /v1/models"),
|
||||
choiceF("GPU class", classes, classes[0]),
|
||||
textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} {
|
||||
if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) {
|
||||
return nil, fmt.Errorf("%s is a whole number", what)
|
||||
}
|
||||
}
|
||||
args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4],
|
||||
"--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]}
|
||||
if v[1] == "0" {
|
||||
args = append(args, "--on-demand")
|
||||
}
|
||||
if v[3] != "" {
|
||||
args = append(args, "--description="+v[3])
|
||||
if proj >= 0 && !strings.HasPrefix(v[proj], "none") {
|
||||
args = append(args, "--projector="+v[proj])
|
||||
}
|
||||
return m.runAfter(args...), nil
|
||||
}})
|
||||
|
||||
Reference in new issue
Block a user