From 54db7aecb7eb455886163962efae1fe9ae77805f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 01:03:48 +0200 Subject: [PATCH] Plan a repository's files against the fleet, and serve the one that suits it the way it fits The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 ++ docs/console.md | 20 +- internal/cli/model_serve.go | 29 +- internal/cli/modelplan.go | 107 +++++- internal/cli/server.go | 1 + internal/cli/tui_actions_test.go | 49 ++- internal/cli/tui_console.go | 2 +- internal/cli/tui_hf.go | 228 ++++++----- internal/control/adminapi/api.go | 52 +++ internal/control/modelserve/modelserve.go | 167 ++++++-- .../control/modelserve/modelserve_test.go | 59 ++- internal/control/modelserve/plan.go | 175 +++++++++ internal/hf/hf.go | 17 +- internal/modelplan/options.go | 356 ++++++++++++++++++ internal/modelplan/options_test.go | 99 +++++ 15 files changed, 1224 insertions(+), 158 deletions(-) create mode 100644 internal/control/modelserve/plan.go create mode 100644 internal/modelplan/options.go create mode 100644 internal/modelplan/options_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d1575..25b810e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,27 @@ is not an update. A model from `server.yaml` is reported but not changed: its source is edited there. +### The file that suits the hardware, and how to spread it + +`llmgw ctl model plan REPO` now plans the files the repository actually has +against the fleet as it is, before anything is downloaded: the model's shape +comes from one GGUF file's header, read from the registry, or from its +config.json. Every file is listed with the best way it runs here — one card; +several of a node's cards (llama.cpp splits by layer, vLLM by tensor +parallelism, dividing the KV heads); several nodes' cards over one fabric, +pipeline-parallel between them (vLLM); or a CPU node's memory — what each +card or node would hold and whether that hardware is free now. Best first: +free now, on a GPU, the fewest cards, then the better quantisation up to +eight bits. `--precision` keeps the old plan in the abstract. + +`llmgw ctl model serve` without `--quant` takes the plan's file, and without +`--cpu` or `--gpu-class`/`--gpus`/`--nodes` the plan's way of running it — +a GGUF model on GPUs is now given a GPU profile rather than always the CPU. +`--fallbacks` writes the next best ways too, lower in priority, so placement +uses one of them when the first is busy. The console's Serve a model shows +the plan for each file, marks the best, and asks for the way and the +fallback. + ### Models of several files: vision projectors, original weights - A vision model's projector (`mmproj`) is fetched beside its GGUF weights diff --git a/docs/console.md b/docs/console.md index 978db9c..86742d7 100644 --- a/docs/console.md +++ b/docs/console.md @@ -188,13 +188,21 @@ to restart it. Another control plane keeps its own copy. - **Serve a model** — Hugging Face's models: trending, most downloaded, most liked or newest (**S** changes which), or **/** to search. **F** switches between GGUF files, which llama.cpp runs on the CPU or a GPU, and - original safetensors weights, which vLLM runs on GPUs. Enter on a GGUF - repository lists its files with their sizes; a vision model offers its - projector too, which is fetched beside it and loaded with it so the model - reads images. Enter on original weights offers them with their config and - tokenizer, a GPU class and how many GPUs a replica spans. What is chosen is + original safetensors weights, which vLLM runs on GPUs. Enter on a + repository plans it against the cluster as it is, before anything is + fetched: the model's shape is read from one file's header, or its + config.json, and every file is listed with the best way it runs here — on + one card, on several of a node's cards, across several nodes' cards over a + fabric (original weights only), or on a CPU node — what each card or node + would hold, and whether that hardware is free now. The file that suits the + cluster best is first, marked ★; one that fits nowhere says so. Choosing a + file offers every way it fits, best first, and what to do when that one is + busy: the next best way, written as a lower-priority profile so placement + falls back to it, or wait. A vision model offers its projector too, fetched + beside it and loaded with it so the model reads images. What is chosen is fetched into the mirror, checked, added to the catalog and started, and a - replica is asked to say hello (`llmgw ctl model serve`). + replica is asked to say hello (`llmgw ctl model serve`; the plan alone is + `llmgw ctl model plan`). `mirror.registry` points it at another registry, for a site without the internet. - **Upgrade the cluster** to a release: this machine, then every node in turn diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 79e9c27..c7dc715 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" ) @@ -25,27 +24,26 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { "that the nodes' llama.cpp can load it and sizes its memory from its own\n" + "header, adds it to the catalog, waits for a replica, and asks it to say\n" + "hello. Each step is shown as it happens; the first that fails stops it.\n\n" + + "Without --quant or --file, the file is the one that suits the cluster best,\n" + + "and where it runs is the plan's best way for it: on one card, on several of\n" + + "a node's, over several nodes on a fabric, or on a CPU node (`llmgw ctl model\n" + + "plan REPO` lists them). --cpu or --gpu-class with --gpus and --nodes say it\n" + + "instead; --fallbacks adds the next best ways for when the first is busy.\n\n" + "A vision model's projector (--projector) is fetched beside it and loaded with\n" + "it. --format safetensors serves the original weights instead, with their\n" + - "config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" + + "config.json and tokenizer, on vLLM.\n\n" + " llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" + " llmgw ctl model serve owner/big-GGUF --on-demand\n" + " llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" + - " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g", + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --fallbacks\n" + + " llmgw ctl model serve meta-llama/Llama-3.1-405B-Instruct --format safetensors --gpu-class h100-80g --gpus 8 --nodes 2", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { req.Repo = args[0] if onDemand { req.Replicas = 0 } - // One kind of GPU in the fleet: that one, unless told. - if req.Format == "safetensors" && req.GPUClass == "" { - var sum adminapi.Summary - if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 { - req.GPUClass = sum.GPUs[0].Class - } - } body, _ := json.Marshal(req) var job modelserve.Job if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { @@ -55,12 +53,17 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { }, } f := c.Flags() - f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)") + f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)") f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant") f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it") f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs") - f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)") - f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans") + f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says") + f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says") + f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)") + f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)") + f.IntVar(&req.TP, "tp", 0, "the older name of --gpus") + _ = f.MarkHidden("tp") + f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy") f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index eba0746..cb5eb16 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -2,11 +2,15 @@ package cli import ( "fmt" + "net/url" + "strconv" "strings" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) @@ -26,13 +30,20 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { context int params int64 precs []string + format string ) c := &cobra.Command{ Use: "plan ", - Short: "Work out whether a model fits this cluster, and how it should be split", - Long: "Reads the model's architecture from its repository and the hardware from\n" + - "the running fleet, then prints every way it could run — fewest GPUs\n" + - "first — and a catalog entry for the best one.\n\n" + + Short: "Work out which of a model's files suits this cluster, and how it should be split", + Long: "Lists every file the repository has — each GGUF quantisation, or with\n" + + "--format safetensors its original weights — with the best way each could\n" + + "run here: on one card, on several of a node's, over several nodes on a\n" + + "fabric, or on a CPU node; what each card would hold and whether the cards\n" + + "are free now. The model's shape is read from one file's header, nothing is\n" + + "downloaded. `llmgw ctl model serve` without --quant takes the first.\n\n" + + "With --precision it plans in the abstract instead, from the model's\n" + + "config: every way it could run at bf16, fp8, q8_0 or q4_k_m, fewest GPUs\n" + + "first, and a catalog entry for the best one.\n\n" + "Every candidate is listed rather than one answer, because the choice is\n" + "not the planner's to make: with cards to spare you want the\n" + "full-precision fit, and with one card free you want to know a quantised\n" + @@ -40,6 +51,11 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { repo := args[0] + // The files the repository has, against the fleet as it is — + // unless a precision or a size is given to plan in the abstract. + if len(precs) == 0 && params == 0 { + return planFiles(cmd, client, repo, format, context) + } m, err := modelplan.FetchModel(cmd.Context(), repo) if err != nil { return err @@ -112,7 +128,9 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { } c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)") c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one") - c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)") + c.Flags().StringSliceVar(&precs, "precision", nil, "plan in the abstract, for these precisions (bf16, fp8, q8_0, q4_k_m), "+ + "from the model's config rather than its files") + c.Flags().StringVar(&format, "format", "gguf", "gguf files, for llama.cpp, or safetensors, the original weights for vLLM") return c } @@ -238,3 +256,82 @@ func humanBytes(n int64) string { return fmt.Sprintf("%.0f KiB", float64(n)/1024) } } + +// planFiles prints the control plane's plan of a repository's files. +func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string, context int) error { + q := url.Values{"repo": {repo}, "format": {format}} + if context > 0 { + q.Set("context", strconv.Itoa(context)) + } + var p modelserve.Plan + if err := client().get(cmd.Context(), "/api/v1/plan?"+q.Encode(), &p); err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%s at %s", p.Repo, catalog.ShortRevision(p.Revision)) + if p.Arch != "" { + fmt.Fprintf(out, ", %s", p.Arch) + } + if p.Params > 0 { + fmt.Fprintf(out, ", %.1fB parameters", float64(p.Params)/1e9) + } + fmt.Fprintf(out, "; planned for a %d-token context\n", p.Context) + if p.Gated { + fmt.Fprintln(out, "gated: the mirror cannot fetch it without an account's approval") + } + if !p.Servable { + fmt.Fprintf(out, "the nodes' engine cannot run it: %s\n", p.Reason) + return errDoesNotFit + } + if p.Note != "" { + fmt.Fprintln(out, "note: "+p.Note) + } + fmt.Fprintln(out) + w := tabTo(out) + fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW") + for _, a := range p.Artifacts { + name := a.Name + if a.Shards > 1 && a.Format == "gguf" { + name += fmt.Sprintf(" (%d parts)", a.Shards) + } + if a.Best == nil { + fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes)) + continue + } + b := a.Best + now := "busy" + if b.FreeNow { + now = "free" + } + holds := fmt.Sprintf("%.1f GB a card", b.NeedGB) + if b.CPU { + holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now) + } + _ = w.Flush() + rec, ok := p.Recommended() + if !ok { + fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.") + return errDoesNotFit + } + fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label()) + if len(p.Options) > 1 { + fmt.Fprintln(out, "\nEvery way, best first:") + w = tabTo(out) + for _, o := range p.Options { + now := "busy" + if o.FreeNow { + now = "free" + } + fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now) + } + _ = w.Flush() + } + fmt.Fprintf(out, "\nServe it: llmgw ctl model serve %s", p.Repo) + if p.Format == "safetensors" { + fmt.Fprint(out, " --format safetensors") + } + fmt.Fprintln(out, " --fallbacks") + return nil +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 2a79d45..6b7698c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -334,6 +334,7 @@ func newServerCmd() *cobra.Command { // changes it, and what replaces replicas. Current: func() []catalog.Model { return admin.Catalog().Models() }, Models: catalogSvc, Restart: admin.Reconciler().Restart, + Pool: adminAPI.PoolNodes, }) adminAPI.SetServe(serve) updateMetrics.Set(serve) diff --git a/internal/cli/tui_actions_test.go b/internal/cli/tui_actions_test.go index 75e777e..0091454 100644 --- a/internal/cli/tui_actions_test.go +++ b/internal/cli/tui_actions_test.go @@ -14,6 +14,8 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -283,7 +285,41 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil { t.Fatal(err) } - m := newConsoleModel("127.0.0.1:9090", cfgPath) + // The control plane's plan of the repository's files. + plan := modelserve.Plan{Repo: "bartowski/Qwen2.5-3B-Instruct-GGUF", Revision: "abc", Format: "gguf", Context: 16384, + Servable: true, Artifacts: []modelserve.PlannedArtifact{ + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q4_K_M.gguf", Format: "gguf", Bytes: 2 << 30}}, + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q8_0.gguf", Format: "gguf", Bytes: 3 << 30}}, + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-F16.gguf", Format: "gguf", Bytes: 6 << 30}}, + }} + gpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1, + NeedGB: 5, HaveGB: 21.6, Hosts: 1, FreeNow: true} + cpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 5, HaveGB: 27, Hosts: 2, FreeNow: true} + for _, o := range []struct { + art string + o modelplan.Option + }{{"Qwen2.5-3B-Instruct-Q8_0.gguf", gpu}, {"Qwen2.5-3B-Instruct-Q4_K_M.gguf", gpu}, + {"Qwen2.5-3B-Instruct-Q8_0.gguf", cpu}, {"Qwen2.5-3B-Instruct-F16.gguf", cpu}} { + o.o.Artifact = o.art + plan.Options = append(plan.Options, o.o) + } + for i := range plan.Artifacts { + for j := range plan.Options { + if plan.Options[j].Artifact == plan.Artifacts[i].Name { + plan.Artifacts[i].Best = &plan.Options[j] + break + } + } + } + admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo { + _ = json.NewEncoder(w).Encode(plan) + return + } + _, _ = w.Write([]byte("[]")) + })) + defer admin.Close() + m := newConsoleModel(admin.URL, cfgPath) m.loaded, m.menu, m.inPanel = true, cServe, true var tm tea.Model = m @@ -303,19 +339,22 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) tm = run(tm, cmd) d := tm.(consoleModel).menuD - if d == nil || len(d.items) != 2 { + if d == nil || len(d.items) != 3 || !strings.Contains(d.items[0].label, "★") || + !strings.Contains(d.items[0].label, "Q8_0") || !strings.Contains(d.items[0].label, "1 × 6000-24g") || + !strings.Contains(d.items[2].label, "CPU") { t.Fatalf("files offered: %+v", d) } - tm = press(tm, "down", "enter") + tm = press(tm, "enter") f := tm.(consoleModel).formD - if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" { + if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" || len(f.fields[4].options) != 2 { t.Fatalf("form: %+v", f) } f.focus = len(f.fields) tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) act := tm.(consoleModel).action got := strings.Join(act.targets, " ") - if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") { + if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") || + !strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") { t.Fatalf("would run %q %q", act.kind, got) } } diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 94fbd08..8059171 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -304,7 +304,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.dialog, m.dialogTitle = "error", "Hugging Face" m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key") } else { - m.menuD = m.repoMenu(msg.repo) + m.menuD = m.repoMenu(msg.plan) } case consolePeopleMsg: m.people = peopleData(msg) diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 18eab3f..06d8e7c 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -3,6 +3,8 @@ package cli import ( "context" "fmt" + "net/url" + "slices" "strconv" "strings" "sync" @@ -13,6 +15,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // Serving from Hugging Face: the registry's GGUF models, trending or @@ -43,7 +46,7 @@ type consoleHFMsg struct { } type consoleRepoMsg struct { - repo hf.Repo + plan modelserve.Plan err error } @@ -79,142 +82,171 @@ func (m consoleModel) fetchHF() tea.Cmd { } } +// fetchRepo asks the control plane for a repository's plan: its files, and +// how each could run on this cluster. func (m *consoleModel) fetchRepo(id string) tea.Cmd { - m.busy = "Reading " + id - c := m.registry() + m.busy = "Planning " + id + c := m.client + format := "gguf" + if m.hf.safetensors { + format = "safetensors" + } return func() tea.Msg { - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - r, err := c.Repo(ctx, id) - return consoleRepoMsg{r, err} + var p modelserve.Plan + err := c.get(ctx, "/api/v1/plan?"+url.Values{"repo": {id}, "format": {format}}.Encode(), &p) + return consoleRepoMsg{p, err} } } -// repoMenu offers a repository's GGUF files, smallest first as listed. -func (m consoleModel) repoMenu(r hf.Repo) *menuDialog { - var items []menuItem - if m.hf.safetensors { - title := r.ID - set, err := modelserve.SafetensorsSet(r) - if err != nil { - return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}} - } - return &menuDialog{title: title, items: []menuItem{{ - fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)), - func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil }, - }}} +// optionLabel is one way to run a file, with what it needs and whether the +// hardware is free. +func optionLabel(o modelplan.Option) string { + now := sDim.Render("busy now") + if o.FreeNow { + now = sOK.Render("free") } - for _, set := range hf.Group(r.Files) { - if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") { - continue + need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB) + if o.CPU { + need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB) + } + return fmt.Sprintf("%s · %s · %s", o.Label(), need, now) +} + +// repoMenu offers a repository's files, each with the best way it runs here; +// the plan's choice first. +func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog { + title := p.Repo + if p.Gated { + title += " · gated" + } + var items []menuItem + if !p.Servable { + return &menuDialog{title: title, items: []menuItem{{"The nodes' engine cannot run it: " + p.Reason, + func(*consoleModel) tea.Cmd { return nil }}}} + } + rec, hasRec := p.Recommended() + arts := append([]modelserve.PlannedArtifact{}, p.Artifacts...) + // The recommended file first, then the rest as the repository lists them. + for i, a := range arts { + if hasRec && a.Name == rec.Artifact && i > 0 { + arts = append([]modelserve.PlannedArtifact{a}, append(arts[:i:i], arts[i+1:]...)...) + break } - low := strings.ToLower(set.Name) - // Projectors and imatrix data are not a model to serve on their own. - if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") { - continue + } + for _, a := range arts { + name := a.Name + if a.Format == "safetensors" { + name = "original weights" } - label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes)) - if set.Shards > 1 { - label += fmt.Sprintf(" %d parts", set.Shards) + label := fmt.Sprintf("%-40s %8s ", truncate(name, 40), bytesShort(a.Bytes)) + switch { + case a.Best == nil: + label += sBad.Render("fits nowhere") + default: + label += optionLabel(*a.Best) } - set := set + if hasRec && a.Name == rec.Artifact { + label = sOK.Render("★ ") + label + } else { + label = " " + label + } + a := a items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd { - m.serveForm(r, set) + if a.Best == nil { + m.dialog, m.dialogTitle = "error", a.Name + m.dialogText = fmt.Sprintf("No card holds %s with a %d-token context, nor does any node's memory.\n"+ + "A smaller quantisation, or more cards free.\n\n%s", a.Name, p.Context, sDim.Render("any key")) + return nil + } + m.serveForm(p, a) return nil }}) } - title := r.ID - if len(items) == 0 { - items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}} - } - if r.Gated.Is() { - title += " · gated" - } return &menuDialog{title: title, items: items} } -func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { +func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) { server := "--server=" + m.client.base - note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes)) - if r.Architecture != "" { - note += ", " + r.Architecture + var opts []modelplan.Option + for _, o := range p.Options { + if o.Artifact == a.Name { + opts = append(opts, o) + } } - if r.License != "" { - note += ", licence " + r.License + var where []string + for _, o := range opts { + where = append(where, optionLabel(o)) + } + note := fmt.Sprintf("%s, %s", a.Name, bytesShort(a.Bytes)) + if a.Format == "safetensors" { + note = fmt.Sprintf("Original weights, %s in %d files with their config and tokenizer, for vLLM", bytesShort(a.Bytes), a.Shards) + } + if p.Arch != "" { + note += ", " + p.Arch + } + if p.License != "" { + note += ", licence " + p.License + } + if p.Note != "" { + note += "\n" + sWarn.Render("! ") + p.Note + } + note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+ + "catalog, and a replica is started and asked to say hello.", p.Context) + ctxDefault := "4096" + if a.Format == "safetensors" { + ctxDefault = strconv.Itoa(p.Context) } - note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes." fields := []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("name", hf.ModelName(p.Repo), "what clients ask for"), textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "4096", "what one request may use"), + textF("context length", ctxDefault, "what one request may use"), textF("description", "", "optional: shown to clients in /v1/models"), + choiceF("runs on", where, where[0]), + choiceF("when that is busy", []string{"the next best way", "wait for it"}, "the next best way"), } - // A vision model: its projector, loaded beside it, is what reads images. - if proj := modelserve.Projectors(r); len(proj) > 0 { - fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0])) + fields[4].hint = "every way this file fits, best first" + proj := -1 + if a.Format == "gguf" && len(p.Projectors) > 0 { + fields = append(fields, choiceF("images", append(append([]string{}, p.Projectors...), "none: text only"), p.Projectors[0])) fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model" + proj = len(fields) - 1 } - m.openForm(&formDialog{title: "Serve " + r.ID, note: note, - fields: fields, + m.openForm(&formDialog{title: "Serve " + p.Repo, note: note, fields: fields, submit: func(m *consoleModel, v []string) (tea.Cmd, error) { n, err := strconv.Atoi(v[1]) if err != nil || n < 0 { return nil, fmt.Errorf("replicas is a whole number") } - if _, err := strconv.Atoi(v[2]); err != nil { + if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 { return nil, fmt.Errorf("the context length is a whole number") } - args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name, - "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} + args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1], + "--context-length=" + v[2]} + if a.Format == "safetensors" { + args = append(args, "--format=safetensors") + } else { + args = append(args, "--file="+a.Name) + } + o := opts[slices.Index(where, v[4])] + if o.CPU { + args = append(args, "--cpu") + } else { + args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode), + "--nodes="+strconv.Itoa(max(o.Nodes, 1))) + } + if strings.HasPrefix(v[5], "the next") && len(opts) > 1 { + args = append(args, "--fallbacks") + } if n == 0 { args = append(args, "--on-demand") } if v[3] != "" { args = append(args, "--description="+v[3]) } - if len(v) > 4 && !strings.HasPrefix(v[4], "none") { - args = append(args, "--projector="+v[4]) - } - return m.runAfter(args...), nil - }}) -} - -func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) { - server := "--server=" + m.client.base - var classes []string - for _, g := range m.data.sum.GPUs { - classes = append(classes, g.Class) - } - if len(classes) == 0 { - m.dialog, m.dialogTitle = "error", "No GPUs" - m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" + - "F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key") - return - } - m.openForm(&formDialog{title: "Serve " + r.ID, - note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+ - "Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)), - fields: []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), - textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "8192", "what one request may use; vLLM's --max-model-len"), - textF("description", "", "optional: shown to clients in /v1/models"), - choiceF("GPU class", classes, classes[0]), - textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"), - }, - submit: func(m *consoleModel, v []string) (tea.Cmd, error) { - for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} { - if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) { - return nil, fmt.Errorf("%s is a whole number", what) - } - } - args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4], - "--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} - if v[1] == "0" { - args = append(args, "--on-demand") - } - if v[3] != "" { - args = append(args, "--description="+v[3]) + if proj >= 0 && !strings.HasPrefix(v[proj], "none") { + args = append(args, "--projector="+v[proj]) } return m.runAfter(args...), nil }}) diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index fbf4db4..6ad85dd 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" @@ -126,6 +127,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin) + mux.HandleFunc("GET /api/v1/plan", a.planModel) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -1544,6 +1546,56 @@ func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision}) } +// PoolNodes is the fleet's ready nodes as a plan sees them: their cards, +// which of them are free, the fabric they share, and their memory. +func (a *API) PoolNodes() []modelplan.PoolNode { + var out []modelplan.PoolNode + for _, n := range a.Nodes() { + if n.State != "ready" { + continue + } + p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB, + HostFreeGB: n.MemoryGB} + if len(n.Classes) > 0 { + p.Class = n.Classes[0] + p.GPUMemGB = modelplan.MemFromClass(p.Class) + } + if len(n.Fabrics) > 0 { + p.Fabric = n.Fabrics[0] + } + if l := n.Load; l != nil && l.MemTotal > 0 { + p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30) + } + out = append(out, p) + } + return out +} + +func (a *API) planModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "planning needs a mirror and its registry, and this control plane lacks one") + return + } + q := r.URL.Query() + req := modelserve.PlanRequest{Repo: q.Get("repo"), Format: q.Get("format")} + if c := q.Get("context"); c != "" { + n, err := strconv.Atoi(c) + if err != nil || n < 0 { + writeError(w, http.StatusBadRequest, "context is a whole number of tokens") + return + } + req.Context = n + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + p, err := a.serve.Plan(ctx, req) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, p) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 8cda83b..a80cf5a 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -30,6 +30,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // Registry reads a repository. @@ -77,6 +78,10 @@ type Deps struct { Current func() []catalog.Model Models Models Restart func(ids ...string) error + + // Pool is the fleet's ready nodes as a plan sees them: what decides + // which file suits the hardware, and how it is spread. + Pool func() []modelplan.PoolNode } // Request is what to serve. @@ -93,9 +98,18 @@ type Request struct { // Format is "gguf", run by llama.cpp, or "safetensors": the original // weights with their config and tokenizer, run by vLLM on GPUs of // GPUClass, across TP of them. - Format string `json:"format,omitempty"` - GPUClass string `json:"gpu_class,omitempty"` - TP int `json:"tp,omitempty"` + Format string `json:"format,omitempty"` + // Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on + // each of Nodes nodes. Left empty, the plan decides: the best way the + // cluster can run the chosen file. TP is GPUsPerNode's older name. + CPU bool `json:"cpu,omitempty"` + GPUClass string `json:"gpu_class,omitempty"` + GPUsPerNode int `json:"gpus_per_node,omitempty"` + Nodes int `json:"nodes,omitempty"` + TP int `json:"tp,omitempty"` + // Fallbacks adds the next best ways, lower in priority, so a model whose + // first choice is busy runs on another rather than waiting. + Fallbacks bool `json:"fallbacks,omitempty"` // Name is the catalog name; the repository suggests one otherwise. Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` @@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) { case "", "gguf": req.Format = "gguf" case "safetensors": - if req.GPUClass == "" { - return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it") - } if req.Projector != "" { return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files") } - req.TP = max(req.TP, 1) + if req.CPU { + return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU") + } default: return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) } - if req.Quant == "" && req.File == "" { - req.Quant = "Q4_K_M" - } if req.CtxPool <= 0 { req.CtxPool = 16384 } @@ -274,6 +284,30 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo)) return } + if req.TP > 0 && req.GPUsPerNode == 0 { + req.GPUsPerNode = req.TP + } + // The plan: which file suits the hardware, and how it spreads over it. + planCtx := req.CtxPool + if req.Format == "safetensors" { + planCtx = req.ContextLength + } + var plan Plan + planned := false + if s.deps.Pool != nil { + if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil { + planned = true + } else { + s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err) + } + } + if req.Format == "gguf" && req.Quant == "" && req.File == "" { + if rec, ok := plan.Recommended(); planned && ok { + req.File = rec.Artifact + } else { + req.Quant = "Q4_K_M" + } + } pickSet := pick if req.Format == "safetensors" { pickSet = pickSafetensors @@ -283,6 +317,11 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", err) return } + layout, err := chooseLayout(req, plan, planned, set) + if err != nil { + s.fail(j, "registry", err) + return + } if req.Projector != "" { i := slices.IndexFunc(repo.Files, func(f hf.File) bool { return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) && @@ -303,8 +342,12 @@ func (s *Server) run(j *Job) { s.mu.Lock() j.Model = name s.mu.Unlock() - s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s", - set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision())) + where := make([]string, 0, len(layout)) + for _, o := range layout { + where = append(where, o.Label()) + } + s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s", + set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "))) // 2. The mirror: fetch, and check what arrived against what was published. source, err := s.mirror(ctx, j, repo, set, name) @@ -313,15 +356,12 @@ func (s *Server) run(j *Job) { return } - // 3. Whether the nodes' llama.cpp loads it, and how much memory it needs. + // 3. Whether the nodes' engine loads it, and how much memory it needs. if req.Format == "safetensors" { // vLLM reads the model's shape from its config.json, which came - // with the weights; the GPUs it needs are the class and width asked. - s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass)) - s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ - Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100, - Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)}, - }) + // with the weights. + s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes))) + s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0)) return } s.step(j, "check", Running, "reading its header from the mirror") @@ -351,22 +391,99 @@ func (s *Server) run(j *Job) { cores = catalog.SuggestCores(nc.NodeCPUs(memGB)) } } + s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores)) +} - s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ - Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, - }) +// chooseLayout is where a model runs: what was asked, else the plan's best +// for the chosen file; with Fallbacks, the next best of other shapes after. +func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) { + artifact := set.Name + engine := "llamacpp" + if req.Format == "safetensors" { + artifact, engine = "original", "vllm" + } + var opts []modelplan.Option + for _, o := range plan.Options { + if o.Artifact == artifact { + opts = append(opts, o) + } + } + var out []modelplan.Option + switch { + case req.CPU || req.GPUClass != "": + o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU} + if !req.CPU { + o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1) + } + if o.Nodes > 1 && engine == "llamacpp" { + return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that") + } + out = append(out, o) + case len(opts) > 0: + out = append(out, opts[0]) + case planned: + return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+ + "nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo) + case engine == "vllm": + return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class") + default: + out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true}) + } + if req.Fallbacks { + shape := func(o modelplan.Option) string { + return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes) + } + seen := map[string]bool{shape(out[0]): true} + for _, o := range opts { + if len(out) == 3 { + break + } + if !seen[shape(o)] { + seen[shape(o)] = true + out = append(out, o) + } + } + } + return out, nil +} + +// profilesFor turns the ways chosen into catalog profiles, the first +// preferred. +func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile { + var out []catalog.Profile + for i, o := range layout { + p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i} + switch { + case o.CPU: + p.Name = "cpu" + p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + p.MinMemoryGB, p.CPUCores = memGB, cores + case o.Engine == "llamacpp": + p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class) + p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode + p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + default: + p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class) + p.GPUClass, p.TP = o.Class, o.GPUsPerNode + if o.Nodes > 1 { + p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode + } + p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)} + } + out = append(out, p) + } + return out } // addAndStart puts the model in the catalog with its profile, waits for a // replica and asks it to answer. -func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) { +func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) { req := j.Request // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, - Profiles: []catalog.Profile{prof}, + Profiles: profs, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go index 2760628..fb5e419 100644 --- a/internal/control/modelserve/modelserve_test.go +++ b/internal/control/modelserve/modelserve_test.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // ggufHeader is enough of a GGUF file for its architecture and shape. @@ -38,7 +39,12 @@ func ggufHeader() []byte { return b.Bytes() } -type fakeRegistry struct{ repo hf.Repo } +type fakeRegistry struct { + repo hf.Repo + base string +} + +func (f fakeRegistry) Base() string { return f.base } func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil } @@ -265,8 +271,10 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) {Path: "original/consolidated.safetensors", Bytes: 7 << 30}, } s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"}) - if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil { - t.Error("safetensors with no GPU class was taken") + // With no fleet to plan against, the GPUs must be named. + job0, _ := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", Name: "x"}) + if j := wait(t, s, job0.ID); j.State != Failed || !strings.Contains(j.Error, "GPU class") { + t.Errorf("safetensors with no GPU class: %+v", j) } job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1}) if err != nil { @@ -289,3 +297,48 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) t.Errorf("%s %+v", mod.Source, p) } } + +func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) { + files := []hf.File{ + {Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "a"}, + {Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"}, + {Path: "m-F16.gguf", Bytes: 40 << 30, SHA256: "c"}, + } + s, m, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "a", "m-Q8_0.gguf": "b"}) + s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL} + s.deps.Pool = func() []modelplan.PoolNode { + return []modelplan.PoolNode{ + {ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 60}, + {ID: "c", HostMemGB: 16, HostFreeGB: 14}, + } + } + p, err := s.Plan(context.Background(), PlanRequest{Repo: "o/m-GGUF"}) + if err != nil { + t.Fatal(err) + } + rec, _ := p.Recommended() + if rec.Artifact != "m-Q8_0.gguf" || rec.CPU || rec.Class != "6000-24g" || p.Artifacts[2].Best == nil || !p.Artifacts[2].Best.CPU { + t.Fatalf("plan: %+v; f16: %+v", rec, p.Artifacts[2].Best) + } + job, err := s.Start(Request{Repo: "o/m-GGUF", Replicas: 1, Fallbacks: true}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + if m.got[0].File != "m-Q8_0.gguf" { + t.Errorf("fetched %s", m.got[0].File) + } + profs := c.added[0].Profiles + if len(profs) != 2 || profs[0].GPUClass != "6000-24g" || profs[0].GPUsPerNode != 1 || profs[0].Engine != "llamacpp" || + profs[1].Name != "cpu" || profs[1].Priority >= profs[0].Priority || profs[1].MinMemoryGB == 0 { + t.Errorf("profiles: %+v", profs) + } + // A file too big for any card or any node's memory: refused, saying so. + s.deps.Pool = func() []modelplan.PoolNode { return []modelplan.PoolNode{{ID: "c", HostMemGB: 16, HostFreeGB: 14}} } + job, _ = s.Start(Request{Repo: "o/m-GGUF", File: "m-F16.gguf", Name: "big"}) + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "fits nowhere") { + t.Errorf("%+v", j) + } +} diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go new file mode 100644 index 0000000..aef850b --- /dev/null +++ b/internal/control/modelserve/plan.go @@ -0,0 +1,175 @@ +package modelserve + +import ( + "context" + "errors" + "fmt" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" +) + +// Planning: which of a repository's files suits the cluster's hardware, and +// how — one card, several of a node's, several nodes' over a fabric, or a +// CPU node — worked out from the files themselves before any is fetched. +// The model's shape is read from one GGUF file's header, a few megabytes +// from the registry, or from its config.json; the hardware is the fleet as +// it is now. + +// PlanRequest is what to plan. +type PlanRequest struct { + Repo string `json:"repo"` + Format string `json:"format,omitempty"` // gguf (default) or safetensors + // Context is the token pool: llama.cpp's --ctx-size, or vLLM's + // --max-model-len. 0 takes 16384 and 8192. + Context int `json:"context,omitempty"` +} + +// PlannedArtifact is one file a repository offers, and the best way to run it. +type PlannedArtifact struct { + modelplan.Artifact + Best *modelplan.Option `json:"best,omitempty"` +} + +// Plan is every way a repository's files could run here, best first. +type Plan struct { + Repo string `json:"repo"` + Revision string `json:"revision"` + Format string `json:"format"` + Arch string `json:"architecture,omitempty"` + License string `json:"license,omitempty"` + Params int64 `json:"params,omitempty"` + Context int `json:"context"` + Gated bool `json:"gated,omitempty"` + // Servable is false when the nodes' engine cannot run the model at all, + // and Reason says why. + Servable bool `json:"servable"` + Reason string `json:"reason,omitempty"` + // Note is anything the plan could not know, said once. + Note string `json:"note,omitempty"` + Projectors []string `json:"projectors,omitempty"` + Artifacts []PlannedArtifact `json:"artifacts"` + Options []modelplan.Option `json:"options"` +} + +// Recommended is the best option of all, if any runs. +func (p Plan) Recommended() (modelplan.Option, bool) { + if len(p.Options) == 0 { + return modelplan.Option{}, false + } + return p.Options[0], true +} + +// ggufSets are a repository's GGUF weights: models, not projectors or +// importance matrices. +func ggufSets(repo hf.Repo) []hf.FileSet { + var weights []hf.File + for _, f := range repo.Files { + lower := strings.ToLower(f.Path) + if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") && !strings.Contains(lower, "imatrix") { + weights = append(weights, f) + } + } + return hf.Group(weights) +} + +// Plan works out where a repository's files could run. +func (s *Server) Plan(ctx context.Context, req PlanRequest) (Plan, error) { + if s.deps.Pool == nil { + return Plan{}, errors.New("this control plane cannot see its fleet here") + } + req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/") + if !repoPattern.MatchString(req.Repo) { + return Plan{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo) + } + repo, err := s.deps.Registry.Repo(ctx, req.Repo) + if err != nil { + return Plan{}, err + } + return s.plan(ctx, repo, req) +} + +func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, error) { + p := Plan{Repo: repo.ID, Revision: repo.Revision, Format: req.Format, Arch: repo.Architecture, + License: repo.License, Params: repo.Params, Context: req.Context, Gated: repo.Gated.Is(), Servable: true, + Projectors: Projectors(repo)} + var arts []modelplan.Artifact + var shape modelplan.Shape + switch req.Format { + case "", "gguf": + p.Format = "gguf" + if p.Context <= 0 { + p.Context = 16384 + } + sets := ggufSets(repo) + if len(sets) == 0 { + return p, fmt.Errorf("%s has no GGUF files", repo.ID) + } + small := sets[0] + for _, st := range sets { + arts = append(arts, modelplan.Artifact{Name: st.Name, Format: "gguf", Bytes: st.Bytes, + Shards: st.Shards, Quant: modelplan.QuantOf(st.Name)}) + if st.Bytes < small.Bytes { + small = st + } + } + // Every quantisation of a model has its shape: the smallest file's + // header says it, read from the registry without fetching the file. + base := "" + if b, ok := s.deps.Registry.(interface{ Base() string }); ok { + base = b.Base() + } + v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion) + if v.Checked && !v.Servable { + p.Servable, p.Reason = false, v.Reason + } + shape.KVBytesPerToken = v.Header.KVBytesPerToken() + shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count + if p.Arch == "" { + p.Arch = v.Header.Architecture + } + case "safetensors": + if p.Context <= 0 { + p.Context = 8192 + } + set, err := pickSafetensors(repo, Request{}) + if err != nil { + return p, err + } + arts = append(arts, modelplan.Artifact{Name: "original", Format: "safetensors", Bytes: set.Bytes, + Shards: len(set.Files), Quant: "bf16"}) + m, err := modelplan.FetchModel(ctx, repo.ID) + if err == nil { + shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) + shape.KVHeads = m.KVHeads + if p.Params == 0 { + p.Params = m.Params + } + } else { + p.Note = "its config.json could not be read: " + err.Error() + } + default: + return p, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) + } + if shape.KVBytesPerToken == 0 && p.Note == "" { + p.Note = "the model's shape was not readable, so the context's memory is not counted" + } + shape.Params = p.Params + p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context) + if !p.Servable { + p.Options = nil + } + for _, a := range arts { + pa := PlannedArtifact{Artifact: a} + for i := range p.Options { + if p.Options[i].Artifact == a.Name { + pa.Best = &p.Options[i] + break + } + } + p.Artifacts = append(p.Artifacts, pa) + } + return p, nil +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index f67d548..f68adb6 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -326,6 +326,9 @@ type Repo struct { // repository's weights, empty for a repository with none. What decides // whether llama.cpp can load it, known before anything is downloaded. Architecture string + // Params is the parameter count the registry read from the weights, 0 + // when it did not. + Params int64 // From the model card's header: what somebody asks before reading the // rest of it. @@ -346,6 +349,9 @@ func (r Repo) ShortRevision() string { return r.Revision } +// Base is the registry's address, for building a file's download URL. +func (c *Client) Base() string { return c.base } + // Repo reads a repository's revision and file list. func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") } @@ -363,7 +369,11 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { Gated Gated `json:"gated"` GGUF *struct { Architecture string `json:"architecture"` + Total int64 `json:"total"` } `json:"gguf"` + Safetensors *struct { + Total int64 `json:"total"` + } `json:"safetensors"` CardData struct { License string `json:"license"` BaseModel stringish `json:"base_model"` @@ -376,7 +386,7 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { if rev != "" { at = "/revision/" + url.PathEscape(rev) } - if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { + if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=safetensors&expand[]=cardData", &meta); err != nil { return Repo{}, err } @@ -403,7 +413,10 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { License: meta.CardData.License, BaseModel: meta.CardData.BaseModel, Languages: meta.CardData.Language} if meta.GGUF != nil { - r.Architecture = meta.GGUF.Architecture + r.Architecture, r.Params = meta.GGUF.Architecture, meta.GGUF.Total + } + if meta.Safetensors != nil && meta.Safetensors.Total > 0 { + r.Params = meta.Safetensors.Total } if r.ID == "" { r.ID = id diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go new file mode 100644 index 0000000..d34f2d7 --- /dev/null +++ b/internal/modelplan/options.go @@ -0,0 +1,356 @@ +package modelplan + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Options: the ways the files a repository actually has could run on the +// cluster as it is — each file on one card, on several of a node's cards, on +// several nodes' cards over a fabric, or on a CPU node — with what each card +// would hold, and whether the cards are free now or only exist. + +// Artifact is something a repository offers to serve: one GGUF file (or one +// split into parts), or its original safetensors weights. +type Artifact struct { + // Name is the file, or the set's name; for original weights, "original". + Name string `json:"name"` + Format string `json:"format"` // "gguf" or "safetensors" + Bytes int64 `json:"bytes"` + Shards int `json:"shards,omitempty"` + // Quant is the quantisation read from the name, like Q4_K_M; "bf16" for + // original weights. + Quant string `json:"quant,omitempty"` +} + +// Shape is what sizes a model's KV cache, and constrains how it splits. +type Shape struct { + // KVBytesPerToken is one token of context, the whole model. + KVBytesPerToken int64 + // KVHeads must divide by a tensor-parallel width; 0 is unknown. + KVHeads int + // Params is the parameter count, for bits per weight; 0 is unknown. + Params int64 +} + +// PoolNode is one ready node, as a plan sees it. +type PoolNode struct { + ID string + Class string // "" for a node with no GPU + Fabric string + GPUMemGB int + GPUs int + FreeGPUs int + // HostMemGB is its RAM, and HostFreeGB what is unused now. + HostMemGB int + HostFreeGB int +} + +// Option is one way to run one artifact. +type Option struct { + Artifact string `json:"artifact"` + Format string `json:"format"` + Engine string `json:"engine"` + CPU bool `json:"cpu,omitempty"` + Class string `json:"gpu_class,omitempty"` + Fabric string `json:"fabric,omitempty"` + // GPUsPerNode cards on each of Nodes nodes. + GPUsPerNode int `json:"gpus_per_node,omitempty"` + Nodes int `json:"nodes,omitempty"` + // NeedGB is what one card holds, or for CPU the whole replica in RAM; + // HaveGB what a card, or the node, offers for it. + NeedGB float64 `json:"need_gb"` + HaveGB float64 `json:"have_gb"` + // Hosts is how many nodes could run it at all, and FreeNow whether one + // (or enough, together) could right now. + Hosts int `json:"hosts"` + FreeNow bool `json:"free_now"` + // BitsPerWeight is the artifact's precision, 0 when unknown. + BitsPerWeight float64 `json:"bits_per_weight,omitempty"` +} + +// GPUs is every card one replica takes. +func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) } + +// Label says where it runs, in a few words. +func (o Option) Label() string { + if o.CPU { + return "CPU" + } + if o.Nodes > 1 { + return fmt.Sprintf("%d nodes × %d %s on %s", o.Nodes, o.GPUsPerNode, o.Class, o.Fabric) + } + return fmt.Sprintf("%d × %s", o.GPUsPerNode, o.Class) +} + +// gpuShare is how much of a card a plan may count on, as in Fits. +const gpuShare = usableFraction + +// hostShare is how much of a node's RAM one replica may be planned to use: +// the system and the agent keep the rest. +const hostShare = 0.85 + +// ggufOverhead is llama.cpp's own cost beyond weights and cache, as the GGUF +// header's MemoryBytes counts it: compute buffers and the server. +func ggufOverhead(weights int64) int64 { return 1<<30 + weights/20 } + +// Options lists every way each artifact could run, best first. +// +// context is the token pool: llama.cpp's --ctx-size, or vLLM's +// --max-model-len for one sequence. +func Options(arts []Artifact, sh Shape, nodes []PoolNode, context int) []Option { + var out []Option + for _, a := range arts { + bpw := quantBits[strings.ToUpper(a.Quant)] + if sh.Params > 0 { + bpw = float64(a.Bytes) * 8 / float64(sh.Params) + } + kv := int64(context) * sh.KVBytesPerToken + switch a.Format { + case "gguf": + out = append(out, ggufOptions(a, kv, nodes, bpw)...) + case "safetensors": + out = append(out, vllmOptions(a, kv, sh, nodes, bpw)...) + } + } + Rank(out) + return out +} + +func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { + var out []Option + // The CPU: the whole replica in a node's RAM. + total := a.Bytes + kv + ggufOverhead(a.Bytes) + cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true, + NeedGB: GB(total), BitsPerWeight: bpw} + for _, n := range nodes { + have := float64(n.HostMemGB) * hostShare + if n.HostMemGB <= 0 || GB(total) > have { + continue + } + cpu.Hosts++ + cpu.HaveGB = max(cpu.HaveGB, have) + if float64(n.HostFreeGB) >= GB(total) { + cpu.FreeNow = true + } + } + if cpu.Hosts > 0 { + out = append(out, cpu) + } + // GPUs: one node's cards, the model split across them by layer. + for _, c := range classes(nodes) { + budget := float64(c.mem) * gpuShare + for width := 1; width <= c.maxGPUs; width++ { + per := (a.Bytes+kv)/int64(width) + ggufOverhead(a.Bytes/int64(width)) + if GB(per) > budget { + continue + } + o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class, + GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw} + for _, n := range nodes { + if n.Class == c.class && n.GPUs >= width { + o.Hosts++ + if n.FreeGPUs >= width { + o.FreeNow = true + } + } + } + out = append(out, o) + break // more cards than the fewest that hold it buys nothing here + } + } + return out +} + +func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) []Option { + var out []Option + perCard := func(width int) int64 { + return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width)) + } + // Tensor parallel within a node must divide the KV heads. + divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) } + for _, c := range classes(nodes) { + budget := float64(c.mem) * gpuShare + opt := func(perNode, n int, fabric string) Option { + return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric, + GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw} + } + // One node: the narrowest power of two that holds it. + single := false + for w := 1; w <= c.maxGPUs && divides(w); w *= 2 { + if GB(perCard(w)) > budget { + continue + } + o := opt(w, 1, "") + for _, fab := range c.fabrics { + for _, nd := range fab.nodes { + if nd.GPUs >= w { + o.Hosts++ + o.FreeNow = o.FreeNow || nd.FreeGPUs >= w + } + } + } + out = append(out, o) + single = true + break + } + if single { + continue + } + // Several nodes, a node's full width on each, pipeline-parallel + // between them: only within one fabric. + w := 1 + for w*2 <= c.maxGPUs { + w *= 2 + } + if !divides(w) { + continue + } + for _, fab := range c.fabrics { + if fab.name == "" { + continue + } + for n := 2; n <= len(fab.nodes); n *= 2 { + if GB(perCard(w*n)) > budget { + continue + } + o := opt(w, n, fab.name) + free := 0 + for _, nd := range fab.nodes { + if nd.GPUs >= w { + o.Hosts++ + } + if nd.FreeGPUs >= w { + free++ + } + } + if o.Hosts < n { + break + } + o.FreeNow = free >= n + out = append(out, o) + break + } + } + } + return out +} + +type classPool struct { + class string + mem int + maxGPUs int + fabrics []fabricPool +} + +type fabricPool struct { + name string + nodes []PoolNode +} + +// classes groups GPU nodes by card, and each by fabric: a replica spans +// nodes only within one. +func classes(nodes []PoolNode) []classPool { + by := map[string]*classPool{} + var order []string + for _, n := range nodes { + if n.Class == "" || n.GPUs == 0 || n.GPUMemGB == 0 { + continue + } + c := by[n.Class] + if c == nil { + c = &classPool{class: n.Class, mem: n.GPUMemGB} + by[n.Class] = c + order = append(order, n.Class) + } + c.maxGPUs = max(c.maxGPUs, n.GPUs) + i := -1 + for j := range c.fabrics { + if c.fabrics[j].name == n.Fabric { + i = j + } + } + if i < 0 { + c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric}) + i = len(c.fabrics) - 1 + } + c.fabrics[i].nodes = append(c.fabrics[i].nodes, n) + } + sort.Strings(order) + out := make([]classPool, 0, len(order)) + for _, k := range order { + out = append(out, *by[k]) + } + return out +} + +// quantBits is roughly what each quantisation spends a weight, for when the +// parameter count is not known to work it out from the file's size. +var quantBits = map[string]float64{ + "F32": 32, "F16": 16, "BF16": 16, "FP16": 16, "Q8_0": 8.5, "Q6_K": 6.6, + "Q5_K_M": 5.7, "Q5_K_S": 5.5, "Q5_0": 5.5, "Q5_1": 6, "Q4_K_M": 4.8, "Q4_K_S": 4.6, "Q4_0": 4.5, "Q4_1": 5, + "IQ4_XS": 4.3, "IQ4_NL": 4.5, "Q3_K_L": 4.3, "Q3_K_M": 3.9, "Q3_K_S": 3.5, "IQ3_M": 3.7, "IQ3_XS": 3.3, + "IQ3_XXS": 3.1, "Q2_K": 3.0, "IQ2_M": 2.7, "IQ2_XS": 2.3, "IQ2_XXS": 2.1, "IQ1_M": 1.8, "IQ1_S": 1.6, +} + +// qualityCap is the precision past which a bigger file is not a better +// model worth its memory: eight bits keep what sixteen do, near enough. +const qualityCap = 8.6 + +func quality(o Option) float64 { + if o.BitsPerWeight == 0 { + return 0 + } + return min(o.BitsPerWeight, qualityCap) +} + +// Rank orders options best first: what can start now; on a GPU before the +// CPU; the fewest cards, then nodes; then the better precision, up to eight +// bits; then the most room left. +func Rank(opts []Option) { + sort.SliceStable(opts, func(i, j int) bool { + a, b := opts[i], opts[j] + if a.FreeNow != b.FreeNow { + return a.FreeNow + } + if a.CPU != b.CPU { + return !a.CPU + } + if a.GPUs() != b.GPUs() { + return a.GPUs() < b.GPUs() + } + if a.Nodes != b.Nodes { + return a.Nodes < b.Nodes + } + if qa, qb := quality(a), quality(b); qa != qb { + return qa > qb + } + return a.HaveGB-a.NeedGB > b.HaveGB-b.NeedGB + }) +} + +var quantPattern = regexp.MustCompile(`(?i)(IQ\d_[A-Z]+|Q\d_K_[SML]|Q\d_K|Q\d_\d|Q\d|BF16|F16|F32|FP16)`) + +// QuantOf reads a quantisation out of a file name. +func QuantOf(name string) string { + m := quantPattern.FindAllString(name, -1) + if len(m) == 0 { + return "" + } + return strings.ToUpper(m[len(m)-1]) +} + +// MemFromClass reads the card size out of a class label such as "h100-80g". +func MemFromClass(class string) int { + i := strings.LastIndex(class, "-") + if i < 0 || !strings.HasSuffix(class, "g") { + return 0 + } + var gb int + if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil { + return 0 + } + return gb +} diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go new file mode 100644 index 0000000..4551b1b --- /dev/null +++ b/internal/modelplan/options_test.go @@ -0,0 +1,99 @@ +package modelplan + +import ( + "strings" + "testing" +) + +const gib = int64(1) << 30 + +// A 14B GGUF: 48 layers, 8 KV heads of 128 — 192 KB a token at f16. +var shape14B = Shape{KVBytesPerToken: 48 * 8 * 256 * 2, KVHeads: 8, Params: 14_800_000_000} + +func labels(opts []Option) []string { + var out []string + for _, o := range opts { + out = append(out, o.Artifact+" "+o.Label()) + } + return out +} + +func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) { + nodes := []PoolNode{ + {ID: "gpu1", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 40}, + {ID: "cpu1", HostMemGB: 32, HostFreeGB: 30}, + } + arts := []Artifact{ + {Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib}, + {Name: "q8.gguf", Format: "gguf", Bytes: 15 * gib}, + {Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib}, + } + opts := Options(arts, shape14B, nodes, 16384) + got := strings.Join(labels(opts), "; ") + // The best: the better precision on the one card, free now. + if opts[0].Artifact != "q8.gguf" || opts[0].CPU || !opts[0].FreeNow { + t.Errorf("first: %+v (all: %s)", opts[0], got) + } + // f16 fits no card, only a CPU node with the RAM. + for _, o := range opts { + if o.Artifact == "f16.gguf" && !o.CPU { + t.Errorf("f16 placed on a card: %+v", o) + } + } + if !strings.Contains(got, "f16.gguf CPU") { + t.Errorf("f16 not offered the CPU: %s", got) + } + // The card busy: what is free comes first, which is the CPU. + nodes[0].FreeGPUs = 0 + opts = Options(arts, shape14B, nodes, 16384) + if !opts[0].CPU || !opts[0].FreeNow { + t.Errorf("with the card busy: %+v", opts[0]) + } +} + +func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "l4-24g", GPUMemGB: 24, GPUs: 4, FreeGPUs: 4}} + opts := Options([]Artifact{{Name: "70b-q4.gguf", Format: "gguf", Bytes: 40 * gib}}, + Shape{KVBytesPerToken: 80 * 8 * 256 * 2}, nodes, 8192) + if len(opts) != 1 || opts[0].GPUsPerNode != 3 || opts[0].Nodes != 1 { + t.Fatalf("%+v", opts) + } +} + +func TestOriginalWeightsSpanNodesOnlyOverAFabric(t *testing.T) { + var nodes []PoolNode + for _, id := range []string{"a", "b"} { + nodes = append(nodes, PoolNode{ID: id, Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"}) + } + llama70 := Shape{KVBytesPerToken: 80 * 8 * 256 * 2, KVHeads: 8, Params: 70e9} + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib}}, llama70, nodes, 8192) + if len(opts) != 1 || opts[0].GPUsPerNode != 4 || opts[0].Nodes != 1 || opts[0].Engine != "vllm" { + t.Fatalf("70B: %+v", opts) + } + big := Shape{KVBytesPerToken: 126 * 8 * 256 * 2, KVHeads: 8, Params: 405e9} + opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192) + if len(opts) != 1 || opts[0].Nodes != 2 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" || !opts[0].FreeNow { + t.Fatalf("405B: %+v", opts) + } + // No fabric, no replica across the two. + for i := range nodes { + nodes[i].Fabric = "" + } + if opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192); len(opts) != 0 { + t.Fatalf("spanned nodes with no fabric: %+v", opts) + } +} + +func TestQuantisationsAreReadFromNames(t *testing.T) { + for name, want := range map[string]string{ + "Qwen2.5-7B-Instruct-Q4_K_M.gguf": "Q4_K_M", + "qwen2.5-coder-14b-instruct-q4_k_m-00001-of-00002.gguf": "Q4_K_M", + "phi-4-IQ3_XS.gguf": "IQ3_XS", + "model-f16.gguf": "F16", + "JiRackUltra_1b.Q6_K.gguf": "Q6_K", + } { + if got := QuantOf(name); got != want { + t.Errorf("%s: %s", name, got) + } + } +}