diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d1575..25b810e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,27 @@ is not an update. A model from `server.yaml` is reported but not changed: its source is edited there. +### The file that suits the hardware, and how to spread it + +`llmgw ctl model plan REPO` now plans the files the repository actually has +against the fleet as it is, before anything is downloaded: the model's shape +comes from one GGUF file's header, read from the registry, or from its +config.json. Every file is listed with the best way it runs here — one card; +several of a node's cards (llama.cpp splits by layer, vLLM by tensor +parallelism, dividing the KV heads); several nodes' cards over one fabric, +pipeline-parallel between them (vLLM); or a CPU node's memory — what each +card or node would hold and whether that hardware is free now. Best first: +free now, on a GPU, the fewest cards, then the better quantisation up to +eight bits. `--precision` keeps the old plan in the abstract. + +`llmgw ctl model serve` without `--quant` takes the plan's file, and without +`--cpu` or `--gpu-class`/`--gpus`/`--nodes` the plan's way of running it — +a GGUF model on GPUs is now given a GPU profile rather than always the CPU. +`--fallbacks` writes the next best ways too, lower in priority, so placement +uses one of them when the first is busy. The console's Serve a model shows +the plan for each file, marks the best, and asks for the way and the +fallback. + ### Models of several files: vision projectors, original weights - A vision model's projector (`mmproj`) is fetched beside its GGUF weights diff --git a/docs/console.md b/docs/console.md index 978db9c..86742d7 100644 --- a/docs/console.md +++ b/docs/console.md @@ -188,13 +188,21 @@ to restart it. Another control plane keeps its own copy. - **Serve a model** — Hugging Face's models: trending, most downloaded, most liked or newest (**S** changes which), or **/** to search. **F** switches between GGUF files, which llama.cpp runs on the CPU or a GPU, and - original safetensors weights, which vLLM runs on GPUs. Enter on a GGUF - repository lists its files with their sizes; a vision model offers its - projector too, which is fetched beside it and loaded with it so the model - reads images. Enter on original weights offers them with their config and - tokenizer, a GPU class and how many GPUs a replica spans. What is chosen is + original safetensors weights, which vLLM runs on GPUs. Enter on a + repository plans it against the cluster as it is, before anything is + fetched: the model's shape is read from one file's header, or its + config.json, and every file is listed with the best way it runs here — on + one card, on several of a node's cards, across several nodes' cards over a + fabric (original weights only), or on a CPU node — what each card or node + would hold, and whether that hardware is free now. The file that suits the + cluster best is first, marked ★; one that fits nowhere says so. Choosing a + file offers every way it fits, best first, and what to do when that one is + busy: the next best way, written as a lower-priority profile so placement + falls back to it, or wait. A vision model offers its projector too, fetched + beside it and loaded with it so the model reads images. What is chosen is fetched into the mirror, checked, added to the catalog and started, and a - replica is asked to say hello (`llmgw ctl model serve`). + replica is asked to say hello (`llmgw ctl model serve`; the plan alone is + `llmgw ctl model plan`). `mirror.registry` points it at another registry, for a site without the internet. - **Upgrade the cluster** to a release: this machine, then every node in turn diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 79e9c27..c7dc715 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" ) @@ -25,27 +24,26 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { "that the nodes' llama.cpp can load it and sizes its memory from its own\n" + "header, adds it to the catalog, waits for a replica, and asks it to say\n" + "hello. Each step is shown as it happens; the first that fails stops it.\n\n" + + "Without --quant or --file, the file is the one that suits the cluster best,\n" + + "and where it runs is the plan's best way for it: on one card, on several of\n" + + "a node's, over several nodes on a fabric, or on a CPU node (`llmgw ctl model\n" + + "plan REPO` lists them). --cpu or --gpu-class with --gpus and --nodes say it\n" + + "instead; --fallbacks adds the next best ways for when the first is busy.\n\n" + "A vision model's projector (--projector) is fetched beside it and loaded with\n" + "it. --format safetensors serves the original weights instead, with their\n" + - "config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" + + "config.json and tokenizer, on vLLM.\n\n" + " llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" + " llmgw ctl model serve owner/big-GGUF --on-demand\n" + " llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" + - " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g", + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --fallbacks\n" + + " llmgw ctl model serve meta-llama/Llama-3.1-405B-Instruct --format safetensors --gpu-class h100-80g --gpus 8 --nodes 2", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { req.Repo = args[0] if onDemand { req.Replicas = 0 } - // One kind of GPU in the fleet: that one, unless told. - if req.Format == "safetensors" && req.GPUClass == "" { - var sum adminapi.Summary - if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 { - req.GPUClass = sum.GPUs[0].Class - } - } body, _ := json.Marshal(req) var job modelserve.Job if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { @@ -55,12 +53,17 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { }, } f := c.Flags() - f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)") + f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)") f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant") f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it") f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs") - f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)") - f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans") + f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says") + f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says") + f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)") + f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)") + f.IntVar(&req.TP, "tp", 0, "the older name of --gpus") + _ = f.MarkHidden("tp") + f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy") f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index eba0746..cb5eb16 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -2,11 +2,15 @@ package cli import ( "fmt" + "net/url" + "strconv" "strings" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) @@ -26,13 +30,20 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { context int params int64 precs []string + format string ) c := &cobra.Command{ Use: "plan ", - Short: "Work out whether a model fits this cluster, and how it should be split", - Long: "Reads the model's architecture from its repository and the hardware from\n" + - "the running fleet, then prints every way it could run — fewest GPUs\n" + - "first — and a catalog entry for the best one.\n\n" + + Short: "Work out which of a model's files suits this cluster, and how it should be split", + Long: "Lists every file the repository has — each GGUF quantisation, or with\n" + + "--format safetensors its original weights — with the best way each could\n" + + "run here: on one card, on several of a node's, over several nodes on a\n" + + "fabric, or on a CPU node; what each card would hold and whether the cards\n" + + "are free now. The model's shape is read from one file's header, nothing is\n" + + "downloaded. `llmgw ctl model serve` without --quant takes the first.\n\n" + + "With --precision it plans in the abstract instead, from the model's\n" + + "config: every way it could run at bf16, fp8, q8_0 or q4_k_m, fewest GPUs\n" + + "first, and a catalog entry for the best one.\n\n" + "Every candidate is listed rather than one answer, because the choice is\n" + "not the planner's to make: with cards to spare you want the\n" + "full-precision fit, and with one card free you want to know a quantised\n" + @@ -40,6 +51,11 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { repo := args[0] + // The files the repository has, against the fleet as it is — + // unless a precision or a size is given to plan in the abstract. + if len(precs) == 0 && params == 0 { + return planFiles(cmd, client, repo, format, context) + } m, err := modelplan.FetchModel(cmd.Context(), repo) if err != nil { return err @@ -112,7 +128,9 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command { } c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)") c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one") - c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)") + c.Flags().StringSliceVar(&precs, "precision", nil, "plan in the abstract, for these precisions (bf16, fp8, q8_0, q4_k_m), "+ + "from the model's config rather than its files") + c.Flags().StringVar(&format, "format", "gguf", "gguf files, for llama.cpp, or safetensors, the original weights for vLLM") return c } @@ -238,3 +256,82 @@ func humanBytes(n int64) string { return fmt.Sprintf("%.0f KiB", float64(n)/1024) } } + +// planFiles prints the control plane's plan of a repository's files. +func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string, context int) error { + q := url.Values{"repo": {repo}, "format": {format}} + if context > 0 { + q.Set("context", strconv.Itoa(context)) + } + var p modelserve.Plan + if err := client().get(cmd.Context(), "/api/v1/plan?"+q.Encode(), &p); err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%s at %s", p.Repo, catalog.ShortRevision(p.Revision)) + if p.Arch != "" { + fmt.Fprintf(out, ", %s", p.Arch) + } + if p.Params > 0 { + fmt.Fprintf(out, ", %.1fB parameters", float64(p.Params)/1e9) + } + fmt.Fprintf(out, "; planned for a %d-token context\n", p.Context) + if p.Gated { + fmt.Fprintln(out, "gated: the mirror cannot fetch it without an account's approval") + } + if !p.Servable { + fmt.Fprintf(out, "the nodes' engine cannot run it: %s\n", p.Reason) + return errDoesNotFit + } + if p.Note != "" { + fmt.Fprintln(out, "note: "+p.Note) + } + fmt.Fprintln(out) + w := tabTo(out) + fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW") + for _, a := range p.Artifacts { + name := a.Name + if a.Shards > 1 && a.Format == "gguf" { + name += fmt.Sprintf(" (%d parts)", a.Shards) + } + if a.Best == nil { + fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes)) + continue + } + b := a.Best + now := "busy" + if b.FreeNow { + now = "free" + } + holds := fmt.Sprintf("%.1f GB a card", b.NeedGB) + if b.CPU { + holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now) + } + _ = w.Flush() + rec, ok := p.Recommended() + if !ok { + fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.") + return errDoesNotFit + } + fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label()) + if len(p.Options) > 1 { + fmt.Fprintln(out, "\nEvery way, best first:") + w = tabTo(out) + for _, o := range p.Options { + now := "busy" + if o.FreeNow { + now = "free" + } + fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now) + } + _ = w.Flush() + } + fmt.Fprintf(out, "\nServe it: llmgw ctl model serve %s", p.Repo) + if p.Format == "safetensors" { + fmt.Fprint(out, " --format safetensors") + } + fmt.Fprintln(out, " --fallbacks") + return nil +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 2a79d45..6b7698c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -334,6 +334,7 @@ func newServerCmd() *cobra.Command { // changes it, and what replaces replicas. Current: func() []catalog.Model { return admin.Catalog().Models() }, Models: catalogSvc, Restart: admin.Reconciler().Restart, + Pool: adminAPI.PoolNodes, }) adminAPI.SetServe(serve) updateMetrics.Set(serve) diff --git a/internal/cli/tui_actions_test.go b/internal/cli/tui_actions_test.go index 75e777e..0091454 100644 --- a/internal/cli/tui_actions_test.go +++ b/internal/cli/tui_actions_test.go @@ -14,6 +14,8 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -283,7 +285,41 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil { t.Fatal(err) } - m := newConsoleModel("127.0.0.1:9090", cfgPath) + // The control plane's plan of the repository's files. + plan := modelserve.Plan{Repo: "bartowski/Qwen2.5-3B-Instruct-GGUF", Revision: "abc", Format: "gguf", Context: 16384, + Servable: true, Artifacts: []modelserve.PlannedArtifact{ + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q4_K_M.gguf", Format: "gguf", Bytes: 2 << 30}}, + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q8_0.gguf", Format: "gguf", Bytes: 3 << 30}}, + {Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-F16.gguf", Format: "gguf", Bytes: 6 << 30}}, + }} + gpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1, + NeedGB: 5, HaveGB: 21.6, Hosts: 1, FreeNow: true} + cpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 5, HaveGB: 27, Hosts: 2, FreeNow: true} + for _, o := range []struct { + art string + o modelplan.Option + }{{"Qwen2.5-3B-Instruct-Q8_0.gguf", gpu}, {"Qwen2.5-3B-Instruct-Q4_K_M.gguf", gpu}, + {"Qwen2.5-3B-Instruct-Q8_0.gguf", cpu}, {"Qwen2.5-3B-Instruct-F16.gguf", cpu}} { + o.o.Artifact = o.art + plan.Options = append(plan.Options, o.o) + } + for i := range plan.Artifacts { + for j := range plan.Options { + if plan.Options[j].Artifact == plan.Artifacts[i].Name { + plan.Artifacts[i].Best = &plan.Options[j] + break + } + } + } + admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo { + _ = json.NewEncoder(w).Encode(plan) + return + } + _, _ = w.Write([]byte("[]")) + })) + defer admin.Close() + m := newConsoleModel(admin.URL, cfgPath) m.loaded, m.menu, m.inPanel = true, cServe, true var tm tea.Model = m @@ -303,19 +339,22 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) tm = run(tm, cmd) d := tm.(consoleModel).menuD - if d == nil || len(d.items) != 2 { + if d == nil || len(d.items) != 3 || !strings.Contains(d.items[0].label, "★") || + !strings.Contains(d.items[0].label, "Q8_0") || !strings.Contains(d.items[0].label, "1 × 6000-24g") || + !strings.Contains(d.items[2].label, "CPU") { t.Fatalf("files offered: %+v", d) } - tm = press(tm, "down", "enter") + tm = press(tm, "enter") f := tm.(consoleModel).formD - if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" { + if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" || len(f.fields[4].options) != 2 { t.Fatalf("form: %+v", f) } f.focus = len(f.fields) tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) act := tm.(consoleModel).action got := strings.Join(act.targets, " ") - if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --file=Qwen2.5-3B-Instruct-Q8_0.gguf --name=qwen2.5-3b-instruct --replicas=1") { + if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") || + !strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") { t.Fatalf("would run %q %q", act.kind, got) } } diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 94fbd08..8059171 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -304,7 +304,7 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.dialog, m.dialogTitle = "error", "Hugging Face" m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key") } else { - m.menuD = m.repoMenu(msg.repo) + m.menuD = m.repoMenu(msg.plan) } case consolePeopleMsg: m.people = peopleData(msg) diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 18eab3f..06d8e7c 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -3,6 +3,8 @@ package cli import ( "context" "fmt" + "net/url" + "slices" "strconv" "strings" "sync" @@ -13,6 +15,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // Serving from Hugging Face: the registry's GGUF models, trending or @@ -43,7 +46,7 @@ type consoleHFMsg struct { } type consoleRepoMsg struct { - repo hf.Repo + plan modelserve.Plan err error } @@ -79,142 +82,171 @@ func (m consoleModel) fetchHF() tea.Cmd { } } +// fetchRepo asks the control plane for a repository's plan: its files, and +// how each could run on this cluster. func (m *consoleModel) fetchRepo(id string) tea.Cmd { - m.busy = "Reading " + id - c := m.registry() + m.busy = "Planning " + id + c := m.client + format := "gguf" + if m.hf.safetensors { + format = "safetensors" + } return func() tea.Msg { - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - r, err := c.Repo(ctx, id) - return consoleRepoMsg{r, err} + var p modelserve.Plan + err := c.get(ctx, "/api/v1/plan?"+url.Values{"repo": {id}, "format": {format}}.Encode(), &p) + return consoleRepoMsg{p, err} } } -// repoMenu offers a repository's GGUF files, smallest first as listed. -func (m consoleModel) repoMenu(r hf.Repo) *menuDialog { - var items []menuItem - if m.hf.safetensors { - title := r.ID - set, err := modelserve.SafetensorsSet(r) - if err != nil { - return &menuDialog{title: title, items: []menuItem{{err.Error(), func(*consoleModel) tea.Cmd { return nil }}}} - } - return &menuDialog{title: title, items: []menuItem{{ - fmt.Sprintf("Original weights for vLLM: %s in %d files", bytesShort(set.Bytes), len(set.Files)), - func(m *consoleModel) tea.Cmd { m.serveVLLMForm(r, set); return nil }, - }}} +// optionLabel is one way to run a file, with what it needs and whether the +// hardware is free. +func optionLabel(o modelplan.Option) string { + now := sDim.Render("busy now") + if o.FreeNow { + now = sOK.Render("free") } - for _, set := range hf.Group(r.Files) { - if !strings.HasSuffix(strings.ToLower(set.Files[0].Path), ".gguf") { - continue + need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB) + if o.CPU { + need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB) + } + return fmt.Sprintf("%s · %s · %s", o.Label(), need, now) +} + +// repoMenu offers a repository's files, each with the best way it runs here; +// the plan's choice first. +func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog { + title := p.Repo + if p.Gated { + title += " · gated" + } + var items []menuItem + if !p.Servable { + return &menuDialog{title: title, items: []menuItem{{"The nodes' engine cannot run it: " + p.Reason, + func(*consoleModel) tea.Cmd { return nil }}}} + } + rec, hasRec := p.Recommended() + arts := append([]modelserve.PlannedArtifact{}, p.Artifacts...) + // The recommended file first, then the rest as the repository lists them. + for i, a := range arts { + if hasRec && a.Name == rec.Artifact && i > 0 { + arts = append([]modelserve.PlannedArtifact{a}, append(arts[:i:i], arts[i+1:]...)...) + break } - low := strings.ToLower(set.Name) - // Projectors and imatrix data are not a model to serve on their own. - if strings.Contains(low, "mmproj") || strings.Contains(low, "imatrix") { - continue + } + for _, a := range arts { + name := a.Name + if a.Format == "safetensors" { + name = "original weights" } - label := fmt.Sprintf("%-44s %8s", truncate(set.Name, 44), bytesShort(set.Bytes)) - if set.Shards > 1 { - label += fmt.Sprintf(" %d parts", set.Shards) + label := fmt.Sprintf("%-40s %8s ", truncate(name, 40), bytesShort(a.Bytes)) + switch { + case a.Best == nil: + label += sBad.Render("fits nowhere") + default: + label += optionLabel(*a.Best) } - set := set + if hasRec && a.Name == rec.Artifact { + label = sOK.Render("★ ") + label + } else { + label = " " + label + } + a := a items = append(items, menuItem{label, func(m *consoleModel) tea.Cmd { - m.serveForm(r, set) + if a.Best == nil { + m.dialog, m.dialogTitle = "error", a.Name + m.dialogText = fmt.Sprintf("No card holds %s with a %d-token context, nor does any node's memory.\n"+ + "A smaller quantisation, or more cards free.\n\n%s", a.Name, p.Context, sDim.Render("any key")) + return nil + } + m.serveForm(p, a) return nil }}) } - title := r.ID - if len(items) == 0 { - items = []menuItem{{"No GGUF file here to serve", func(*consoleModel) tea.Cmd { return nil }}} - } - if r.Gated.Is() { - title += " · gated" - } return &menuDialog{title: title, items: items} } -func (m *consoleModel) serveForm(r hf.Repo, set hf.FileSet) { +func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) { server := "--server=" + m.client.base - note := fmt.Sprintf("%s, %s", set.Name, bytesShort(set.Bytes)) - if r.Architecture != "" { - note += ", " + r.Architecture + var opts []modelplan.Option + for _, o := range p.Options { + if o.Artifact == a.Name { + opts = append(opts, o) + } } - if r.License != "" { - note += ", licence " + r.License + var where []string + for _, o := range opts { + where = append(where, optionLabel(o)) + } + note := fmt.Sprintf("%s, %s", a.Name, bytesShort(a.Bytes)) + if a.Format == "safetensors" { + note = fmt.Sprintf("Original weights, %s in %d files with their config and tokenizer, for vLLM", bytesShort(a.Bytes), a.Shards) + } + if p.Arch != "" { + note += ", " + p.Arch + } + if p.License != "" { + note += ", licence " + p.License + } + if p.Note != "" { + note += "\n" + sWarn.Render("! ") + p.Note + } + note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+ + "catalog, and a replica is started and asked to say hello.", p.Context) + ctxDefault := "4096" + if a.Format == "safetensors" { + ctxDefault = strconv.Itoa(p.Context) } - note += "\nThe mirror fetches it and checks it, it joins the catalog, and a\nreplica is started and asked to say hello. The output is shown as it goes." fields := []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), + textF("name", hf.ModelName(p.Repo), "what clients ask for"), textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "4096", "what one request may use"), + textF("context length", ctxDefault, "what one request may use"), textF("description", "", "optional: shown to clients in /v1/models"), + choiceF("runs on", where, where[0]), + choiceF("when that is busy", []string{"the next best way", "wait for it"}, "the next best way"), } - // A vision model: its projector, loaded beside it, is what reads images. - if proj := modelserve.Projectors(r); len(proj) > 0 { - fields = append(fields, choiceF("images", append(proj, "none: text only"), proj[0])) + fields[4].hint = "every way this file fits, best first" + proj := -1 + if a.Format == "gguf" && len(p.Projectors) > 0 { + fields = append(fields, choiceF("images", append(append([]string{}, p.Projectors...), "none: text only"), p.Projectors[0])) fields[len(fields)-1].hint = "the vision projector, fetched and loaded beside the model" + proj = len(fields) - 1 } - m.openForm(&formDialog{title: "Serve " + r.ID, note: note, - fields: fields, + m.openForm(&formDialog{title: "Serve " + p.Repo, note: note, fields: fields, submit: func(m *consoleModel, v []string) (tea.Cmd, error) { n, err := strconv.Atoi(v[1]) if err != nil || n < 0 { return nil, fmt.Errorf("replicas is a whole number") } - if _, err := strconv.Atoi(v[2]); err != nil { + if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 { return nil, fmt.Errorf("the context length is a whole number") } - args := []string{"ctl", server, "model", "serve", r.ID, "--file=" + set.Name, - "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} + args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1], + "--context-length=" + v[2]} + if a.Format == "safetensors" { + args = append(args, "--format=safetensors") + } else { + args = append(args, "--file="+a.Name) + } + o := opts[slices.Index(where, v[4])] + if o.CPU { + args = append(args, "--cpu") + } else { + args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode), + "--nodes="+strconv.Itoa(max(o.Nodes, 1))) + } + if strings.HasPrefix(v[5], "the next") && len(opts) > 1 { + args = append(args, "--fallbacks") + } if n == 0 { args = append(args, "--on-demand") } if v[3] != "" { args = append(args, "--description="+v[3]) } - if len(v) > 4 && !strings.HasPrefix(v[4], "none") { - args = append(args, "--projector="+v[4]) - } - return m.runAfter(args...), nil - }}) -} - -func (m *consoleModel) serveVLLMForm(r hf.Repo, set hf.FileSet) { - server := "--server=" + m.client.base - var classes []string - for _, g := range m.data.sum.GPUs { - classes = append(classes, g.Class) - } - if len(classes) == 0 { - m.dialog, m.dialogTitle = "error", "No GPUs" - m.dialogText = "Original weights run on vLLM, on GPUs, and this fleet reports none.\n" + - "F goes back to GGUF models, which run on the CPU.\n\n" + sDim.Render("any key") - return - } - m.openForm(&formDialog{title: "Serve " + r.ID, - note: fmt.Sprintf("%s in %d files: the weights, config.json and the tokenizer.\n"+ - "Fetched into the mirror and checked, added, and run by vLLM on the GPUs chosen.", bytesShort(set.Bytes), len(set.Files)), - fields: []formField{ - textF("name", hf.ModelName(r.ID), "what clients ask for"), - textF("replicas", "1", "0 runs none until it is first asked for"), - textF("context length", "8192", "what one request may use; vLLM's --max-model-len"), - textF("description", "", "optional: shown to clients in /v1/models"), - choiceF("GPU class", classes, classes[0]), - textF("GPUs per replica", "1", "tensor parallel: the cards one replica spans"), - }, - submit: func(m *consoleModel, v []string) (tea.Cmd, error) { - for i, what := range map[int]string{1: "replicas", 2: "the context length", 5: "GPUs per replica"} { - if n, err := strconv.Atoi(v[i]); err != nil || n < 0 || (i == 5 && n < 1) { - return nil, fmt.Errorf("%s is a whole number", what) - } - } - args := []string{"ctl", server, "model", "serve", r.ID, "--format=safetensors", "--gpu-class=" + v[4], - "--tp=" + v[5], "--name=" + v[0], "--replicas=" + v[1], "--context-length=" + v[2]} - if v[1] == "0" { - args = append(args, "--on-demand") - } - if v[3] != "" { - args = append(args, "--description="+v[3]) + if proj >= 0 && !strings.HasPrefix(v[proj], "none") { + args = append(args, "--projector="+v[proj]) } return m.runAfter(args...), nil }}) diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index fbf4db4..6ad85dd 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" @@ -126,6 +127,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin) + mux.HandleFunc("GET /api/v1/plan", a.planModel) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -1544,6 +1546,56 @@ func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision}) } +// PoolNodes is the fleet's ready nodes as a plan sees them: their cards, +// which of them are free, the fabric they share, and their memory. +func (a *API) PoolNodes() []modelplan.PoolNode { + var out []modelplan.PoolNode + for _, n := range a.Nodes() { + if n.State != "ready" { + continue + } + p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB, + HostFreeGB: n.MemoryGB} + if len(n.Classes) > 0 { + p.Class = n.Classes[0] + p.GPUMemGB = modelplan.MemFromClass(p.Class) + } + if len(n.Fabrics) > 0 { + p.Fabric = n.Fabrics[0] + } + if l := n.Load; l != nil && l.MemTotal > 0 { + p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30) + } + out = append(out, p) + } + return out +} + +func (a *API) planModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "planning needs a mirror and its registry, and this control plane lacks one") + return + } + q := r.URL.Query() + req := modelserve.PlanRequest{Repo: q.Get("repo"), Format: q.Get("format")} + if c := q.Get("context"); c != "" { + n, err := strconv.Atoi(c) + if err != nil || n < 0 { + writeError(w, http.StatusBadRequest, "context is a whole number of tokens") + return + } + req.Context = n + } + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + p, err := a.serve.Plan(ctx, req) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, p) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 8cda83b..a80cf5a 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -30,6 +30,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // Registry reads a repository. @@ -77,6 +78,10 @@ type Deps struct { Current func() []catalog.Model Models Models Restart func(ids ...string) error + + // Pool is the fleet's ready nodes as a plan sees them: what decides + // which file suits the hardware, and how it is spread. + Pool func() []modelplan.PoolNode } // Request is what to serve. @@ -93,9 +98,18 @@ type Request struct { // Format is "gguf", run by llama.cpp, or "safetensors": the original // weights with their config and tokenizer, run by vLLM on GPUs of // GPUClass, across TP of them. - Format string `json:"format,omitempty"` - GPUClass string `json:"gpu_class,omitempty"` - TP int `json:"tp,omitempty"` + Format string `json:"format,omitempty"` + // Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on + // each of Nodes nodes. Left empty, the plan decides: the best way the + // cluster can run the chosen file. TP is GPUsPerNode's older name. + CPU bool `json:"cpu,omitempty"` + GPUClass string `json:"gpu_class,omitempty"` + GPUsPerNode int `json:"gpus_per_node,omitempty"` + Nodes int `json:"nodes,omitempty"` + TP int `json:"tp,omitempty"` + // Fallbacks adds the next best ways, lower in priority, so a model whose + // first choice is busy runs on another rather than waiting. + Fallbacks bool `json:"fallbacks,omitempty"` // Name is the catalog name; the repository suggests one otherwise. Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` @@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) { case "", "gguf": req.Format = "gguf" case "safetensors": - if req.GPUClass == "" { - return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it") - } if req.Projector != "" { return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files") } - req.TP = max(req.TP, 1) + if req.CPU { + return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU") + } default: return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) } - if req.Quant == "" && req.File == "" { - req.Quant = "Q4_K_M" - } if req.CtxPool <= 0 { req.CtxPool = 16384 } @@ -274,6 +284,30 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo)) return } + if req.TP > 0 && req.GPUsPerNode == 0 { + req.GPUsPerNode = req.TP + } + // The plan: which file suits the hardware, and how it spreads over it. + planCtx := req.CtxPool + if req.Format == "safetensors" { + planCtx = req.ContextLength + } + var plan Plan + planned := false + if s.deps.Pool != nil { + if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil { + planned = true + } else { + s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err) + } + } + if req.Format == "gguf" && req.Quant == "" && req.File == "" { + if rec, ok := plan.Recommended(); planned && ok { + req.File = rec.Artifact + } else { + req.Quant = "Q4_K_M" + } + } pickSet := pick if req.Format == "safetensors" { pickSet = pickSafetensors @@ -283,6 +317,11 @@ func (s *Server) run(j *Job) { s.fail(j, "registry", err) return } + layout, err := chooseLayout(req, plan, planned, set) + if err != nil { + s.fail(j, "registry", err) + return + } if req.Projector != "" { i := slices.IndexFunc(repo.Files, func(f hf.File) bool { return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) && @@ -303,8 +342,12 @@ func (s *Server) run(j *Job) { s.mu.Lock() j.Model = name s.mu.Unlock() - s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s", - set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision())) + where := make([]string, 0, len(layout)) + for _, o := range layout { + where = append(where, o.Label()) + } + s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s", + set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "))) // 2. The mirror: fetch, and check what arrived against what was published. source, err := s.mirror(ctx, j, repo, set, name) @@ -313,15 +356,12 @@ func (s *Server) run(j *Job) { return } - // 3. Whether the nodes' llama.cpp loads it, and how much memory it needs. + // 3. Whether the nodes' engine loads it, and how much memory it needs. if req.Format == "safetensors" { // vLLM reads the model's shape from its config.json, which came - // with the weights; the GPUs it needs are the class and width asked. - s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass)) - s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ - Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100, - Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)}, - }) + // with the weights. + s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes))) + s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0)) return } s.step(j, "check", Running, "reading its header from the mirror") @@ -351,22 +391,99 @@ func (s *Server) run(j *Job) { cores = catalog.SuggestCores(nc.NodeCPUs(memGB)) } } + s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores)) +} - s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{ - Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, - Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores, - }) +// chooseLayout is where a model runs: what was asked, else the plan's best +// for the chosen file; with Fallbacks, the next best of other shapes after. +func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) { + artifact := set.Name + engine := "llamacpp" + if req.Format == "safetensors" { + artifact, engine = "original", "vllm" + } + var opts []modelplan.Option + for _, o := range plan.Options { + if o.Artifact == artifact { + opts = append(opts, o) + } + } + var out []modelplan.Option + switch { + case req.CPU || req.GPUClass != "": + o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU} + if !req.CPU { + o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1) + } + if o.Nodes > 1 && engine == "llamacpp" { + return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that") + } + out = append(out, o) + case len(opts) > 0: + out = append(out, opts[0]) + case planned: + return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+ + "nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo) + case engine == "vllm": + return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class") + default: + out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true}) + } + if req.Fallbacks { + shape := func(o modelplan.Option) string { + return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes) + } + seen := map[string]bool{shape(out[0]): true} + for _, o := range opts { + if len(out) == 3 { + break + } + if !seen[shape(o)] { + seen[shape(o)] = true + out = append(out, o) + } + } + } + return out, nil +} + +// profilesFor turns the ways chosen into catalog profiles, the first +// preferred. +func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile { + var out []catalog.Profile + for i, o := range layout { + p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i} + switch { + case o.CPU: + p.Name = "cpu" + p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + p.MinMemoryGB, p.CPUCores = memGB, cores + case o.Engine == "llamacpp": + p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class) + p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode + p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + default: + p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class) + p.GPUClass, p.TP = o.Class, o.GPUsPerNode + if o.Nodes > 1 { + p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode + } + p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)} + } + out = append(out, p) + } + return out } // addAndStart puts the model in the catalog with its profile, waits for a // replica and asks it to answer. -func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) { +func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) { req := j.Request // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, - Profiles: []catalog.Profile{prof}, + Profiles: profs, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go index 2760628..fb5e419 100644 --- a/internal/control/modelserve/modelserve_test.go +++ b/internal/control/modelserve/modelserve_test.go @@ -16,6 +16,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" ) // ggufHeader is enough of a GGUF file for its architecture and shape. @@ -38,7 +39,12 @@ func ggufHeader() []byte { return b.Bytes() } -type fakeRegistry struct{ repo hf.Repo } +type fakeRegistry struct { + repo hf.Repo + base string +} + +func (f fakeRegistry) Base() string { return f.base } func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil } @@ -265,8 +271,10 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) {Path: "original/consolidated.safetensors", Bytes: 7 << 30}, } s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"}) - if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil { - t.Error("safetensors with no GPU class was taken") + // With no fleet to plan against, the GPUs must be named. + job0, _ := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", Name: "x"}) + if j := wait(t, s, job0.ID); j.State != Failed || !strings.Contains(j.Error, "GPU class") { + t.Errorf("safetensors with no GPU class: %+v", j) } job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1}) if err != nil { @@ -289,3 +297,48 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) t.Errorf("%s %+v", mod.Source, p) } } + +func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) { + files := []hf.File{ + {Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "a"}, + {Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"}, + {Path: "m-F16.gguf", Bytes: 40 << 30, SHA256: "c"}, + } + s, m, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "a", "m-Q8_0.gguf": "b"}) + s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL} + s.deps.Pool = func() []modelplan.PoolNode { + return []modelplan.PoolNode{ + {ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 60}, + {ID: "c", HostMemGB: 16, HostFreeGB: 14}, + } + } + p, err := s.Plan(context.Background(), PlanRequest{Repo: "o/m-GGUF"}) + if err != nil { + t.Fatal(err) + } + rec, _ := p.Recommended() + if rec.Artifact != "m-Q8_0.gguf" || rec.CPU || rec.Class != "6000-24g" || p.Artifacts[2].Best == nil || !p.Artifacts[2].Best.CPU { + t.Fatalf("plan: %+v; f16: %+v", rec, p.Artifacts[2].Best) + } + job, err := s.Start(Request{Repo: "o/m-GGUF", Replicas: 1, Fallbacks: true}) + if err != nil { + t.Fatal(err) + } + if j := wait(t, s, job.ID); j.State != Done { + t.Fatalf("%s %+v", j.Error, j.Steps) + } + if m.got[0].File != "m-Q8_0.gguf" { + t.Errorf("fetched %s", m.got[0].File) + } + profs := c.added[0].Profiles + if len(profs) != 2 || profs[0].GPUClass != "6000-24g" || profs[0].GPUsPerNode != 1 || profs[0].Engine != "llamacpp" || + profs[1].Name != "cpu" || profs[1].Priority >= profs[0].Priority || profs[1].MinMemoryGB == 0 { + t.Errorf("profiles: %+v", profs) + } + // A file too big for any card or any node's memory: refused, saying so. + s.deps.Pool = func() []modelplan.PoolNode { return []modelplan.PoolNode{{ID: "c", HostMemGB: 16, HostFreeGB: 14}} } + job, _ = s.Start(Request{Repo: "o/m-GGUF", File: "m-F16.gguf", Name: "big"}) + if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "fits nowhere") { + t.Errorf("%+v", j) + } +} diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go new file mode 100644 index 0000000..aef850b --- /dev/null +++ b/internal/control/modelserve/plan.go @@ -0,0 +1,175 @@ +package modelserve + +import ( + "context" + "errors" + "fmt" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" +) + +// Planning: which of a repository's files suits the cluster's hardware, and +// how — one card, several of a node's, several nodes' over a fabric, or a +// CPU node — worked out from the files themselves before any is fetched. +// The model's shape is read from one GGUF file's header, a few megabytes +// from the registry, or from its config.json; the hardware is the fleet as +// it is now. + +// PlanRequest is what to plan. +type PlanRequest struct { + Repo string `json:"repo"` + Format string `json:"format,omitempty"` // gguf (default) or safetensors + // Context is the token pool: llama.cpp's --ctx-size, or vLLM's + // --max-model-len. 0 takes 16384 and 8192. + Context int `json:"context,omitempty"` +} + +// PlannedArtifact is one file a repository offers, and the best way to run it. +type PlannedArtifact struct { + modelplan.Artifact + Best *modelplan.Option `json:"best,omitempty"` +} + +// Plan is every way a repository's files could run here, best first. +type Plan struct { + Repo string `json:"repo"` + Revision string `json:"revision"` + Format string `json:"format"` + Arch string `json:"architecture,omitempty"` + License string `json:"license,omitempty"` + Params int64 `json:"params,omitempty"` + Context int `json:"context"` + Gated bool `json:"gated,omitempty"` + // Servable is false when the nodes' engine cannot run the model at all, + // and Reason says why. + Servable bool `json:"servable"` + Reason string `json:"reason,omitempty"` + // Note is anything the plan could not know, said once. + Note string `json:"note,omitempty"` + Projectors []string `json:"projectors,omitempty"` + Artifacts []PlannedArtifact `json:"artifacts"` + Options []modelplan.Option `json:"options"` +} + +// Recommended is the best option of all, if any runs. +func (p Plan) Recommended() (modelplan.Option, bool) { + if len(p.Options) == 0 { + return modelplan.Option{}, false + } + return p.Options[0], true +} + +// ggufSets are a repository's GGUF weights: models, not projectors or +// importance matrices. +func ggufSets(repo hf.Repo) []hf.FileSet { + var weights []hf.File + for _, f := range repo.Files { + lower := strings.ToLower(f.Path) + if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") && !strings.Contains(lower, "imatrix") { + weights = append(weights, f) + } + } + return hf.Group(weights) +} + +// Plan works out where a repository's files could run. +func (s *Server) Plan(ctx context.Context, req PlanRequest) (Plan, error) { + if s.deps.Pool == nil { + return Plan{}, errors.New("this control plane cannot see its fleet here") + } + req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/") + if !repoPattern.MatchString(req.Repo) { + return Plan{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo) + } + repo, err := s.deps.Registry.Repo(ctx, req.Repo) + if err != nil { + return Plan{}, err + } + return s.plan(ctx, repo, req) +} + +func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, error) { + p := Plan{Repo: repo.ID, Revision: repo.Revision, Format: req.Format, Arch: repo.Architecture, + License: repo.License, Params: repo.Params, Context: req.Context, Gated: repo.Gated.Is(), Servable: true, + Projectors: Projectors(repo)} + var arts []modelplan.Artifact + var shape modelplan.Shape + switch req.Format { + case "", "gguf": + p.Format = "gguf" + if p.Context <= 0 { + p.Context = 16384 + } + sets := ggufSets(repo) + if len(sets) == 0 { + return p, fmt.Errorf("%s has no GGUF files", repo.ID) + } + small := sets[0] + for _, st := range sets { + arts = append(arts, modelplan.Artifact{Name: st.Name, Format: "gguf", Bytes: st.Bytes, + Shards: st.Shards, Quant: modelplan.QuantOf(st.Name)}) + if st.Bytes < small.Bytes { + small = st + } + } + // Every quantisation of a model has its shape: the smallest file's + // header says it, read from the registry without fetching the file. + base := "" + if b, ok := s.deps.Registry.(interface{ Base() string }); ok { + base = b.Base() + } + v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion) + if v.Checked && !v.Servable { + p.Servable, p.Reason = false, v.Reason + } + shape.KVBytesPerToken = v.Header.KVBytesPerToken() + shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count + if p.Arch == "" { + p.Arch = v.Header.Architecture + } + case "safetensors": + if p.Context <= 0 { + p.Context = 8192 + } + set, err := pickSafetensors(repo, Request{}) + if err != nil { + return p, err + } + arts = append(arts, modelplan.Artifact{Name: "original", Format: "safetensors", Bytes: set.Bytes, + Shards: len(set.Files), Quant: "bf16"}) + m, err := modelplan.FetchModel(ctx, repo.ID) + if err == nil { + shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) + shape.KVHeads = m.KVHeads + if p.Params == 0 { + p.Params = m.Params + } + } else { + p.Note = "its config.json could not be read: " + err.Error() + } + default: + return p, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format) + } + if shape.KVBytesPerToken == 0 && p.Note == "" { + p.Note = "the model's shape was not readable, so the context's memory is not counted" + } + shape.Params = p.Params + p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context) + if !p.Servable { + p.Options = nil + } + for _, a := range arts { + pa := PlannedArtifact{Artifact: a} + for i := range p.Options { + if p.Options[i].Artifact == a.Name { + pa.Best = &p.Options[i] + break + } + } + p.Artifacts = append(p.Artifacts, pa) + } + return p, nil +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index f67d548..f68adb6 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -326,6 +326,9 @@ type Repo struct { // repository's weights, empty for a repository with none. What decides // whether llama.cpp can load it, known before anything is downloaded. Architecture string + // Params is the parameter count the registry read from the weights, 0 + // when it did not. + Params int64 // From the model card's header: what somebody asks before reading the // rest of it. @@ -346,6 +349,9 @@ func (r Repo) ShortRevision() string { return r.Revision } +// Base is the registry's address, for building a file's download URL. +func (c *Client) Base() string { return c.base } + // Repo reads a repository's revision and file list. func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") } @@ -363,7 +369,11 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { Gated Gated `json:"gated"` GGUF *struct { Architecture string `json:"architecture"` + Total int64 `json:"total"` } `json:"gguf"` + Safetensors *struct { + Total int64 `json:"total"` + } `json:"safetensors"` CardData struct { License string `json:"license"` BaseModel stringish `json:"base_model"` @@ -376,7 +386,7 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { if rev != "" { at = "/revision/" + url.PathEscape(rev) } - if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { + if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=safetensors&expand[]=cardData", &meta); err != nil { return Repo{}, err } @@ -403,7 +413,10 @@ func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { License: meta.CardData.License, BaseModel: meta.CardData.BaseModel, Languages: meta.CardData.Language} if meta.GGUF != nil { - r.Architecture = meta.GGUF.Architecture + r.Architecture, r.Params = meta.GGUF.Architecture, meta.GGUF.Total + } + if meta.Safetensors != nil && meta.Safetensors.Total > 0 { + r.Params = meta.Safetensors.Total } if r.ID == "" { r.ID = id diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go new file mode 100644 index 0000000..d34f2d7 --- /dev/null +++ b/internal/modelplan/options.go @@ -0,0 +1,356 @@ +package modelplan + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Options: the ways the files a repository actually has could run on the +// cluster as it is — each file on one card, on several of a node's cards, on +// several nodes' cards over a fabric, or on a CPU node — with what each card +// would hold, and whether the cards are free now or only exist. + +// Artifact is something a repository offers to serve: one GGUF file (or one +// split into parts), or its original safetensors weights. +type Artifact struct { + // Name is the file, or the set's name; for original weights, "original". + Name string `json:"name"` + Format string `json:"format"` // "gguf" or "safetensors" + Bytes int64 `json:"bytes"` + Shards int `json:"shards,omitempty"` + // Quant is the quantisation read from the name, like Q4_K_M; "bf16" for + // original weights. + Quant string `json:"quant,omitempty"` +} + +// Shape is what sizes a model's KV cache, and constrains how it splits. +type Shape struct { + // KVBytesPerToken is one token of context, the whole model. + KVBytesPerToken int64 + // KVHeads must divide by a tensor-parallel width; 0 is unknown. + KVHeads int + // Params is the parameter count, for bits per weight; 0 is unknown. + Params int64 +} + +// PoolNode is one ready node, as a plan sees it. +type PoolNode struct { + ID string + Class string // "" for a node with no GPU + Fabric string + GPUMemGB int + GPUs int + FreeGPUs int + // HostMemGB is its RAM, and HostFreeGB what is unused now. + HostMemGB int + HostFreeGB int +} + +// Option is one way to run one artifact. +type Option struct { + Artifact string `json:"artifact"` + Format string `json:"format"` + Engine string `json:"engine"` + CPU bool `json:"cpu,omitempty"` + Class string `json:"gpu_class,omitempty"` + Fabric string `json:"fabric,omitempty"` + // GPUsPerNode cards on each of Nodes nodes. + GPUsPerNode int `json:"gpus_per_node,omitempty"` + Nodes int `json:"nodes,omitempty"` + // NeedGB is what one card holds, or for CPU the whole replica in RAM; + // HaveGB what a card, or the node, offers for it. + NeedGB float64 `json:"need_gb"` + HaveGB float64 `json:"have_gb"` + // Hosts is how many nodes could run it at all, and FreeNow whether one + // (or enough, together) could right now. + Hosts int `json:"hosts"` + FreeNow bool `json:"free_now"` + // BitsPerWeight is the artifact's precision, 0 when unknown. + BitsPerWeight float64 `json:"bits_per_weight,omitempty"` +} + +// GPUs is every card one replica takes. +func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) } + +// Label says where it runs, in a few words. +func (o Option) Label() string { + if o.CPU { + return "CPU" + } + if o.Nodes > 1 { + return fmt.Sprintf("%d nodes × %d %s on %s", o.Nodes, o.GPUsPerNode, o.Class, o.Fabric) + } + return fmt.Sprintf("%d × %s", o.GPUsPerNode, o.Class) +} + +// gpuShare is how much of a card a plan may count on, as in Fits. +const gpuShare = usableFraction + +// hostShare is how much of a node's RAM one replica may be planned to use: +// the system and the agent keep the rest. +const hostShare = 0.85 + +// ggufOverhead is llama.cpp's own cost beyond weights and cache, as the GGUF +// header's MemoryBytes counts it: compute buffers and the server. +func ggufOverhead(weights int64) int64 { return 1<<30 + weights/20 } + +// Options lists every way each artifact could run, best first. +// +// context is the token pool: llama.cpp's --ctx-size, or vLLM's +// --max-model-len for one sequence. +func Options(arts []Artifact, sh Shape, nodes []PoolNode, context int) []Option { + var out []Option + for _, a := range arts { + bpw := quantBits[strings.ToUpper(a.Quant)] + if sh.Params > 0 { + bpw = float64(a.Bytes) * 8 / float64(sh.Params) + } + kv := int64(context) * sh.KVBytesPerToken + switch a.Format { + case "gguf": + out = append(out, ggufOptions(a, kv, nodes, bpw)...) + case "safetensors": + out = append(out, vllmOptions(a, kv, sh, nodes, bpw)...) + } + } + Rank(out) + return out +} + +func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { + var out []Option + // The CPU: the whole replica in a node's RAM. + total := a.Bytes + kv + ggufOverhead(a.Bytes) + cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true, + NeedGB: GB(total), BitsPerWeight: bpw} + for _, n := range nodes { + have := float64(n.HostMemGB) * hostShare + if n.HostMemGB <= 0 || GB(total) > have { + continue + } + cpu.Hosts++ + cpu.HaveGB = max(cpu.HaveGB, have) + if float64(n.HostFreeGB) >= GB(total) { + cpu.FreeNow = true + } + } + if cpu.Hosts > 0 { + out = append(out, cpu) + } + // GPUs: one node's cards, the model split across them by layer. + for _, c := range classes(nodes) { + budget := float64(c.mem) * gpuShare + for width := 1; width <= c.maxGPUs; width++ { + per := (a.Bytes+kv)/int64(width) + ggufOverhead(a.Bytes/int64(width)) + if GB(per) > budget { + continue + } + o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class, + GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw} + for _, n := range nodes { + if n.Class == c.class && n.GPUs >= width { + o.Hosts++ + if n.FreeGPUs >= width { + o.FreeNow = true + } + } + } + out = append(out, o) + break // more cards than the fewest that hold it buys nothing here + } + } + return out +} + +func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) []Option { + var out []Option + perCard := func(width int) int64 { + return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width)) + } + // Tensor parallel within a node must divide the KV heads. + divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) } + for _, c := range classes(nodes) { + budget := float64(c.mem) * gpuShare + opt := func(perNode, n int, fabric string) Option { + return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric, + GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw} + } + // One node: the narrowest power of two that holds it. + single := false + for w := 1; w <= c.maxGPUs && divides(w); w *= 2 { + if GB(perCard(w)) > budget { + continue + } + o := opt(w, 1, "") + for _, fab := range c.fabrics { + for _, nd := range fab.nodes { + if nd.GPUs >= w { + o.Hosts++ + o.FreeNow = o.FreeNow || nd.FreeGPUs >= w + } + } + } + out = append(out, o) + single = true + break + } + if single { + continue + } + // Several nodes, a node's full width on each, pipeline-parallel + // between them: only within one fabric. + w := 1 + for w*2 <= c.maxGPUs { + w *= 2 + } + if !divides(w) { + continue + } + for _, fab := range c.fabrics { + if fab.name == "" { + continue + } + for n := 2; n <= len(fab.nodes); n *= 2 { + if GB(perCard(w*n)) > budget { + continue + } + o := opt(w, n, fab.name) + free := 0 + for _, nd := range fab.nodes { + if nd.GPUs >= w { + o.Hosts++ + } + if nd.FreeGPUs >= w { + free++ + } + } + if o.Hosts < n { + break + } + o.FreeNow = free >= n + out = append(out, o) + break + } + } + } + return out +} + +type classPool struct { + class string + mem int + maxGPUs int + fabrics []fabricPool +} + +type fabricPool struct { + name string + nodes []PoolNode +} + +// classes groups GPU nodes by card, and each by fabric: a replica spans +// nodes only within one. +func classes(nodes []PoolNode) []classPool { + by := map[string]*classPool{} + var order []string + for _, n := range nodes { + if n.Class == "" || n.GPUs == 0 || n.GPUMemGB == 0 { + continue + } + c := by[n.Class] + if c == nil { + c = &classPool{class: n.Class, mem: n.GPUMemGB} + by[n.Class] = c + order = append(order, n.Class) + } + c.maxGPUs = max(c.maxGPUs, n.GPUs) + i := -1 + for j := range c.fabrics { + if c.fabrics[j].name == n.Fabric { + i = j + } + } + if i < 0 { + c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric}) + i = len(c.fabrics) - 1 + } + c.fabrics[i].nodes = append(c.fabrics[i].nodes, n) + } + sort.Strings(order) + out := make([]classPool, 0, len(order)) + for _, k := range order { + out = append(out, *by[k]) + } + return out +} + +// quantBits is roughly what each quantisation spends a weight, for when the +// parameter count is not known to work it out from the file's size. +var quantBits = map[string]float64{ + "F32": 32, "F16": 16, "BF16": 16, "FP16": 16, "Q8_0": 8.5, "Q6_K": 6.6, + "Q5_K_M": 5.7, "Q5_K_S": 5.5, "Q5_0": 5.5, "Q5_1": 6, "Q4_K_M": 4.8, "Q4_K_S": 4.6, "Q4_0": 4.5, "Q4_1": 5, + "IQ4_XS": 4.3, "IQ4_NL": 4.5, "Q3_K_L": 4.3, "Q3_K_M": 3.9, "Q3_K_S": 3.5, "IQ3_M": 3.7, "IQ3_XS": 3.3, + "IQ3_XXS": 3.1, "Q2_K": 3.0, "IQ2_M": 2.7, "IQ2_XS": 2.3, "IQ2_XXS": 2.1, "IQ1_M": 1.8, "IQ1_S": 1.6, +} + +// qualityCap is the precision past which a bigger file is not a better +// model worth its memory: eight bits keep what sixteen do, near enough. +const qualityCap = 8.6 + +func quality(o Option) float64 { + if o.BitsPerWeight == 0 { + return 0 + } + return min(o.BitsPerWeight, qualityCap) +} + +// Rank orders options best first: what can start now; on a GPU before the +// CPU; the fewest cards, then nodes; then the better precision, up to eight +// bits; then the most room left. +func Rank(opts []Option) { + sort.SliceStable(opts, func(i, j int) bool { + a, b := opts[i], opts[j] + if a.FreeNow != b.FreeNow { + return a.FreeNow + } + if a.CPU != b.CPU { + return !a.CPU + } + if a.GPUs() != b.GPUs() { + return a.GPUs() < b.GPUs() + } + if a.Nodes != b.Nodes { + return a.Nodes < b.Nodes + } + if qa, qb := quality(a), quality(b); qa != qb { + return qa > qb + } + return a.HaveGB-a.NeedGB > b.HaveGB-b.NeedGB + }) +} + +var quantPattern = regexp.MustCompile(`(?i)(IQ\d_[A-Z]+|Q\d_K_[SML]|Q\d_K|Q\d_\d|Q\d|BF16|F16|F32|FP16)`) + +// QuantOf reads a quantisation out of a file name. +func QuantOf(name string) string { + m := quantPattern.FindAllString(name, -1) + if len(m) == 0 { + return "" + } + return strings.ToUpper(m[len(m)-1]) +} + +// MemFromClass reads the card size out of a class label such as "h100-80g". +func MemFromClass(class string) int { + i := strings.LastIndex(class, "-") + if i < 0 || !strings.HasSuffix(class, "g") { + return 0 + } + var gb int + if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil { + return 0 + } + return gb +} diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go new file mode 100644 index 0000000..4551b1b --- /dev/null +++ b/internal/modelplan/options_test.go @@ -0,0 +1,99 @@ +package modelplan + +import ( + "strings" + "testing" +) + +const gib = int64(1) << 30 + +// A 14B GGUF: 48 layers, 8 KV heads of 128 — 192 KB a token at f16. +var shape14B = Shape{KVBytesPerToken: 48 * 8 * 256 * 2, KVHeads: 8, Params: 14_800_000_000} + +func labels(opts []Option) []string { + var out []string + for _, o := range opts { + out = append(out, o.Artifact+" "+o.Label()) + } + return out +} + +func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) { + nodes := []PoolNode{ + {ID: "gpu1", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 40}, + {ID: "cpu1", HostMemGB: 32, HostFreeGB: 30}, + } + arts := []Artifact{ + {Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib}, + {Name: "q8.gguf", Format: "gguf", Bytes: 15 * gib}, + {Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib}, + } + opts := Options(arts, shape14B, nodes, 16384) + got := strings.Join(labels(opts), "; ") + // The best: the better precision on the one card, free now. + if opts[0].Artifact != "q8.gguf" || opts[0].CPU || !opts[0].FreeNow { + t.Errorf("first: %+v (all: %s)", opts[0], got) + } + // f16 fits no card, only a CPU node with the RAM. + for _, o := range opts { + if o.Artifact == "f16.gguf" && !o.CPU { + t.Errorf("f16 placed on a card: %+v", o) + } + } + if !strings.Contains(got, "f16.gguf CPU") { + t.Errorf("f16 not offered the CPU: %s", got) + } + // The card busy: what is free comes first, which is the CPU. + nodes[0].FreeGPUs = 0 + opts = Options(arts, shape14B, nodes, 16384) + if !opts[0].CPU || !opts[0].FreeNow { + t.Errorf("with the card busy: %+v", opts[0]) + } +} + +func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "l4-24g", GPUMemGB: 24, GPUs: 4, FreeGPUs: 4}} + opts := Options([]Artifact{{Name: "70b-q4.gguf", Format: "gguf", Bytes: 40 * gib}}, + Shape{KVBytesPerToken: 80 * 8 * 256 * 2}, nodes, 8192) + if len(opts) != 1 || opts[0].GPUsPerNode != 3 || opts[0].Nodes != 1 { + t.Fatalf("%+v", opts) + } +} + +func TestOriginalWeightsSpanNodesOnlyOverAFabric(t *testing.T) { + var nodes []PoolNode + for _, id := range []string{"a", "b"} { + nodes = append(nodes, PoolNode{ID: id, Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"}) + } + llama70 := Shape{KVBytesPerToken: 80 * 8 * 256 * 2, KVHeads: 8, Params: 70e9} + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib}}, llama70, nodes, 8192) + if len(opts) != 1 || opts[0].GPUsPerNode != 4 || opts[0].Nodes != 1 || opts[0].Engine != "vllm" { + t.Fatalf("70B: %+v", opts) + } + big := Shape{KVBytesPerToken: 126 * 8 * 256 * 2, KVHeads: 8, Params: 405e9} + opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192) + if len(opts) != 1 || opts[0].Nodes != 2 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" || !opts[0].FreeNow { + t.Fatalf("405B: %+v", opts) + } + // No fabric, no replica across the two. + for i := range nodes { + nodes[i].Fabric = "" + } + if opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192); len(opts) != 0 { + t.Fatalf("spanned nodes with no fabric: %+v", opts) + } +} + +func TestQuantisationsAreReadFromNames(t *testing.T) { + for name, want := range map[string]string{ + "Qwen2.5-7B-Instruct-Q4_K_M.gguf": "Q4_K_M", + "qwen2.5-coder-14b-instruct-q4_k_m-00001-of-00002.gguf": "Q4_K_M", + "phi-4-IQ3_XS.gguf": "IQ3_XS", + "model-f16.gguf": "F16", + "JiRackUltra_1b.Q6_K.gguf": "Q6_K", + } { + if got := QuantOf(name); got != want { + t.Errorf("%s: %s", name, got) + } + } +}