From 4fe9b4645102d9054d4e12b351b87719f644cbbb Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 20 Oct 2026 08:45:33 +0200 Subject: [PATCH] A plan for a big model: experts spread, any number of nodes, memory said A mixture-of-experts model on more than one card is served with vLLM's --enable-expert-parallel; config.json's experts are read by any of their names, and a model that also reads pictures is sized by its text_config. Tensor parallelism past the KV heads is allowed when it is a multiple of them (Qwen3-235B, four KV heads, runs on eight cards), which the plan had refused. Several nodes are offered in any number, not only powers of two. Each way says what it needs of a card, part by part, and of all its cards; the serve form checks a context longer than planned still fits. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/admin-api.openapi.json | 19 +++++ internal/cli/modelplan.go | 18 +++-- internal/cli/tui_hf.go | 21 ++++-- internal/control/modelserve/modelserve.go | 18 ++++- internal/control/modelserve/plan.go | 7 +- internal/modelplan/fit.go | 10 +-- internal/modelplan/fit_test.go | 13 ++-- internal/modelplan/huggingface.go | 12 ++++ internal/modelplan/huggingface_test.go | 12 ++++ internal/modelplan/options.go | 87 +++++++++++++++++++++-- internal/modelplan/options_test.go | 70 ++++++++++++++++++ internal/modelplan/size.go | 4 ++ 12 files changed, 257 insertions(+), 34 deletions(-) diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a6057f4..7f5a334 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -10516,6 +10516,20 @@ "bits_per_weight": { "type": "number", "description": "BitsPerWeight is the artifact's precision, 0 when unknown." + }, + "weights_gb": { + "type": "number", + "description": "What NeedGB is made of, a card's share (or, on the CPU, the node's RAM): the weights, the context's KV cache, and the engine's own." + }, + "context_gb": { + "type": "number" + }, + "engine_gb": { + "type": "number" + }, + "expert_parallel": { + "type": "boolean", + "description": "ExpertParallel spreads a mixture-of-experts model's experts over the cards, whole, rather than slicing each across them: vLLM's --enable-expert-parallel, for a model of experts on more than one card." } } }, @@ -10620,6 +10634,11 @@ "context": { "type": "integer" }, + "kv_bytes_per_token": { + "type": "integer", + "format": "int64", + "description": "KVBytesPerToken is one token of context, the whole model: what an option needs for another context is worked out from it. 0 is unknown." + }, "gated": { "type": "boolean" }, diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index 6ee413d..429b8a8 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -291,14 +291,14 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string } fmt.Fprintln(out) w := tabTo(out) - fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW") + fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tA CARD\tOF\tALL CARDS\tNOW") for _, a := range p.Artifacts { name := a.Name if a.Shards > 1 && a.Format == "gguf" { name += fmt.Sprintf(" (%d parts)", a.Shards) } if a.Best == nil { - fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes)) + fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\t\n", name, bytesShort(a.Bytes)) continue } b := a.Best @@ -306,11 +306,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string if b.FreeNow { now = "free" } - holds := fmt.Sprintf("%.1f GB a card", b.NeedGB) + holds, all := fmt.Sprintf("%.1f GB", b.NeedGB), fmt.Sprintf("%.0f GB", b.TotalGB()) if b.CPU { - holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB) + holds, all = fmt.Sprintf("%.1f GB of RAM", b.NeedGB), "-" } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, all, now) } _ = w.Flush() rec, ok := p.Recommended() @@ -318,7 +318,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.") return errDoesNotFit } - fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label()) + fmt.Fprintf(out, "\nBest: %s on %s: %s.\n", rec.Artifact, rec.Label(), rec.MemoryText()) + if p.KVBytesPerToken > 0 { + fmt.Fprintf(out, "The context costs %.2f GB per 1000 tokens, the whole model (--context to plan another).\n", + modelplan.GB(p.KVBytesPerToken*1000)) + } if len(p.Options) > 1 { fmt.Fprintln(out, "\nEvery way, best first:") w = tabTo(out) @@ -327,7 +331,7 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string if o.FreeNow { now = "free" } - fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now) + fmt.Fprintf(w, " %s\t%s\t%s\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now) } _ = w.Flush() } diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 337907f..fa2dea4 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -107,11 +107,7 @@ func optionLabel(o modelplan.Option) string { if o.FreeNow { now = sOK.Render("free") } - need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB) - if o.CPU { - need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB) - } - return fmt.Sprintf("%s · %s · %s", o.Label(), need, now) + return fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now) } // repoMenu offers a repository's files, each with the best way it runs here; @@ -197,8 +193,15 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact if p.Note != "" { note += "\n" + sWarn.Render("! ") + p.Note } - note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+ - "catalog, and a replica is started and asked to say hello.", p.Context) + note += fmt.Sprintf("\nMemory for a %d-token context, each way it runs:", p.Context) + for _, o := range opts { + note += "\n " + o.Label() + ": " + o.MemoryText() + } + if p.KVBytesPerToken > 0 { + note += fmt.Sprintf("\nThe context costs %.2f GB per 1000 tokens, the whole model; a longer one is checked\n"+ + "before it is served.", modelplan.GB(p.KVBytesPerToken*1000)) + } + note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello." ctxDefault := "4096" if a.Format == "safetensors" { ctxDefault = strconv.Itoa(p.Context) @@ -234,6 +237,10 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact req.File = a.Name } o := opts[slices.Index(where, v[4])] + // Asked for a context other than planned: does it still fit? + if sized := o.WithContext(p.KVBytesPerToken, ctxLen); !sized.Fits() { + return nil, fmt.Errorf("with a %d-token context it needs %s: a shorter context, or another way", ctxLen, sized.MemoryText()) + } if o.CPU { req.CPU = true } else { diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index eb773b8..d5363b4 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -461,8 +461,17 @@ func (s *Server) run(j *Job) { for _, o := range layout { where = append(where, o.Label()) } - s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s", - set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "))) + // What the first way needs of each card, for the context asked for. + need := "" + if first := layout[0]; first.NeedGB > 0 { + tokens := req.ContextLength + if req.Format != "safetensors" && req.CtxPool > 0 { + tokens = req.CtxPool + } + need = "; needs " + first.WithContext(plan.KVBytesPerToken, tokens).MemoryText() + } + s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s%s", + set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "), need)) // 2. The mirror: fetch, and check what arrived against what was published. source, err := s.mirror(ctx, j, repo, set, name) @@ -622,6 +631,11 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode } p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)} + if o.ExpertParallel { + // The experts whole on each card rather than each sliced + // across them: less traffic between the cards for every token. + p.Args = append(p.Args, "--enable-expert-parallel") + } } out = append(out, p) } diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index aef850b..38fa16d 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -42,7 +42,10 @@ type Plan struct { License string `json:"license,omitempty"` Params int64 `json:"params,omitempty"` Context int `json:"context"` - Gated bool `json:"gated,omitempty"` + // KVBytesPerToken is one token of context, the whole model: what an + // option needs for another context is worked out from it. 0 is unknown. + KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"` + Gated bool `json:"gated,omitempty"` // Servable is false when the nodes' engine cannot run the model at all, // and Reason says why. Servable bool `json:"servable"` @@ -144,6 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, if err == nil { shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) shape.KVHeads = m.KVHeads + shape.Experts = m.Experts if p.Params == 0 { p.Params = m.Params } @@ -157,6 +161,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, p.Note = "the model's shape was not readable, so the context's memory is not counted" } shape.Params = p.Params + p.KVBytesPerToken = shape.KVBytesPerToken p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context) if !p.Servable { p.Options = nil diff --git a/internal/modelplan/fit.go b/internal/modelplan/fit.go index 3dacb53..7175d2c 100644 --- a/internal/modelplan/fit.go +++ b/internal/modelplan/fit.go @@ -155,11 +155,11 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit { if nodes > 1 && h.Fabric == "" { continue } - // The KV heads have to divide too, and they are far fewer than the - // attention heads under grouped-query attention: a model with 8 KV - // heads cannot be split sixteen ways. - if m.KVHeads > 0 && width > m.KVHeads { - break + // The KV heads are split among the cards, or past their number each + // card holds a copy of one: the width divides them or they divide + // it. A model with 8 KV heads splits 2, 4, 8 or 16 ways, not 12. + if m.KVHeads > 0 && m.KVHeads%width != 0 && width%m.KVHeads != 0 { + continue } per := GB(d.PerGPU(width)) diff --git a/internal/modelplan/fit_test.go b/internal/modelplan/fit_test.go index 364ffe4..d7eda45 100644 --- a/internal/modelplan/fit_test.go +++ b/internal/modelplan/fit_test.go @@ -89,13 +89,14 @@ func TestATensorGroupNeverSpansFabrics(t *testing.T) { } } -func TestTensorWidthCannotExceedTheKVHeads(t *testing.T) { - // Grouped-query attention leaves far fewer KV heads than attention heads, - // and an engine refuses a width that cannot divide them. Offering tp=16 for - // a model with 8 KV heads is a plan that fails at launch. +func TestTensorWidthFitsTheKVHeads(t *testing.T) { + // Grouped-query attention leaves far fewer KV heads than attention heads. + // An engine splits them among the cards, or past their number gives each + // card a copy of one: so a width that neither divides them nor is a + // multiple of them fails at launch, and is not offered. for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) { - if f.TP > llama70B.KVHeads { - t.Errorf("tp=%d exceeds the model's %d KV heads", f.TP, llama70B.KVHeads) + if llama70B.KVHeads%f.TP != 0 && f.TP%llama70B.KVHeads != 0 { + t.Errorf("tp=%d does not fit the model's %d KV heads", f.TP, llama70B.KVHeads) } } } diff --git a/internal/modelplan/huggingface.go b/internal/modelplan/huggingface.go index 9c6d381..0b7a94d 100644 --- a/internal/modelplan/huggingface.go +++ b/internal/modelplan/huggingface.go @@ -47,9 +47,20 @@ type hfConfig struct { HeadDim int `json:"head_dim"` MaxPositions int `json:"max_position_embeddings"` ModelType string `json:"model_type"` + // The routed experts of a mixture-of-experts model, by whichever name + // its family gives them: Qwen's and Mixtral's, DeepSeek's, Llama 4's. + NumExperts int `json:"num_experts"` + NRoutedExperts int `json:"n_routed_experts"` + NumLocalExperts int `json:"num_local_experts"` + // TextConfig is where a model that also reads pictures keeps its + // language model's figures (Llama 4, Qwen3-VL, Gemma 3). + TextConfig *hfConfig `json:"text_config"` } func (c hfConfig) toModel(repo string) (Model, error) { + if c.NumHiddenLayers == 0 && c.TextConfig != nil { + c = *c.TextConfig + } if c.NumHiddenLayers == 0 { return Model{}, fmt.Errorf("%s: config.json has no num_hidden_layers; is it a text model?", repo) } @@ -57,6 +68,7 @@ func (c hfConfig) toModel(repo string) (Model, error) { Name: repo[strings.LastIndex(repo, "/")+1:], Layers: c.NumHiddenLayers, MaxContext: c.MaxPositions, + Experts: max(c.NumExperts, c.NRoutedExperts, c.NumLocalExperts), } // Grouped-query attention: when the KV head count is absent the model has diff --git a/internal/modelplan/huggingface_test.go b/internal/modelplan/huggingface_test.go index 94fc5c2..d4c879d 100644 --- a/internal/modelplan/huggingface_test.go +++ b/internal/modelplan/huggingface_test.go @@ -75,3 +75,15 @@ func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) { t.Errorf("Name = %q", m.Name) } } + +func TestAModelThatReadsPicturesIsSizedByItsLanguageModel(t *testing.T) { + m, err := hfConfig{TextConfig: &hfConfig{NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8, + HiddenSize: 5120, NumLocalExperts: 16}}.toModel("meta-llama/Llama-4-Scout-17B-16E-Instruct") + if err != nil || m.Layers != 48 || m.KVHeads != 8 || m.Experts != 16 { + t.Errorf("%+v %v", m, err) + } + ds, _ := hfConfig{NumHiddenLayers: 61, NumAttentionHeads: 128, HiddenSize: 7168, NRoutedExperts: 256}.toModel("deepseek-ai/DeepSeek-V3") + if ds.Experts != 256 { + t.Errorf("DeepSeek's experts: %d", ds.Experts) + } +} diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index bb1217d..deb295e 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -33,6 +33,8 @@ type Shape struct { KVHeads int // Params is the parameter count, for bits per weight; 0 is unknown. Params int64 + // Experts is a mixture-of-experts model's routed experts; 0 for dense. + Experts int } // PoolNode is one ready node, as a plan sees it. @@ -96,11 +98,71 @@ type Option struct { FreeNow bool `json:"free_now"` // BitsPerWeight is the artifact's precision, 0 when unknown. BitsPerWeight float64 `json:"bits_per_weight,omitempty"` + // What NeedGB is made of, a card's share (or, on the CPU, the node's + // RAM): the weights, the context's KV cache, and the engine's own. + WeightsGB float64 `json:"weights_gb,omitempty"` + ContextGB float64 `json:"context_gb,omitempty"` + EngineGB float64 `json:"engine_gb,omitempty"` + // ExpertParallel spreads a mixture-of-experts model's experts over the + // cards, whole, rather than slicing each across them: vLLM's + // --enable-expert-parallel, for a model of experts on more than one card. + ExpertParallel bool `json:"expert_parallel,omitempty"` } // GPUs is every card one replica takes. func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) } +// TotalGB is what one replica takes of every card together, or of the +// node's RAM on the CPU. +func (o Option) TotalGB() float64 { + if o.CPU { + return o.NeedGB + } + return o.NeedGB * float64(o.GPUs()) +} + +// Fits is whether a card, or the node, holds what it needs. +func (o Option) Fits() bool { return o.NeedGB <= o.HaveGB } + +// WithContext is the option for another context: kvPerToken a token's KV +// cache, the whole model; the weights and the engine's share stay. +func (o Option) WithContext(kvPerToken int64, tokens int) Option { + if kvPerToken <= 0 || tokens <= 0 { + return o + } + ctx := GB(int64(tokens) * kvPerToken) + if !o.CPU { + ctx /= float64(max(o.GPUs(), 1)) + } + o.NeedGB += ctx - o.ContextGB + o.ContextGB = ctx + return o +} + +// MemoryText says what it needs, part by part. +func (o Option) MemoryText() string { + parts := fmt.Sprintf("weights %.1f + context %.1f + engine %.1f", o.WeightsGB, o.ContextGB, o.EngineGB) + if o.WeightsGB == 0 && o.ContextGB == 0 { + parts = "" + } + if o.CPU { + s := fmt.Sprintf("%.1f GB of RAM", o.NeedGB) + if parts != "" { + s += " (" + parts + ")" + } + return s + fmt.Sprintf(" of %.0f usable", o.HaveGB) + } + s := fmt.Sprintf("%.1f GB a card", o.NeedGB) + if parts != "" { + s += " (" + parts + ")" + } + s += fmt.Sprintf(" of %.0f usable", o.HaveGB) + if o.GPUs() > 1 { + s += fmt.Sprintf("; %.0f GB on %d cards", o.TotalGB(), o.GPUs()) + } + return s +} + // Label says where it runs, in a few words. func (o Option) Label() string { if o.CPU { @@ -151,7 +213,8 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { // The CPU: the whole replica in a node's RAM. total := a.Bytes + kv + ggufOverhead(a.Bytes) cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true, - NeedGB: GB(total), BitsPerWeight: bpw} + NeedGB: GB(total), BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes), ContextGB: GB(kv), EngineGB: GB(ggufOverhead(a.Bytes))} for _, n := range nodes { have := float64(n.HostMemGB) * hostShare if n.HostMemGB <= 0 || GB(total) > have { @@ -175,7 +238,9 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { continue } o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class, - GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw} + GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes / int64(width)), ContextGB: GB(kv / int64(width)), + EngineGB: GB(ggufOverhead(a.Bytes / int64(width)))} for _, fab := range c.fabrics { for _, n := range fab.nodes { if n.GPUs >= width { @@ -198,13 +263,20 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) perCard := func(width int) int64 { return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width)) } - // Tensor parallel within a node must divide the KV heads. - divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) } + // Tensor parallel splits the KV heads among the cards, or past their + // number gives each card a copy of one: so the width divides the heads, + // or the heads divide the width. Qwen3-235B has four and runs on eight. + divides := func(w int) bool { + return sh.KVHeads == 0 || sh.KVHeads%w == 0 || w%sh.KVHeads == 0 + } for _, c := range classes(nodes) { budget := float64(c.mem) * gpuShare opt := func(perNode, n int, fabric string) Option { + cards := int64(perNode * n) return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric, - GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw} + GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes / cards), ContextGB: GB(kv / cards), EngineGB: GB(overheadBytes(a.Bytes / cards)), + ExpertParallel: sh.Experts > 1 && perNode*n > 1} } // One node: the narrowest power of two that holds it. single := false @@ -243,7 +315,10 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) if fab.name == "" { continue } - for n := 2; n <= len(fab.nodes); n *= 2 { + // Any number of nodes: pipeline stages need not be a power of + // two, and a third node is what a model just too big for two + // needs. + for n := 2; n <= len(fab.nodes); n++ { if GB(perCard(w*n)) > budget { continue } diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go index 476bbe4..768d594 100644 --- a/internal/modelplan/options_test.go +++ b/internal/modelplan/options_test.go @@ -123,3 +123,73 @@ func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) { t.Errorf("GPU: %s first", o[0].Artifact) } } + +// A model of experts spread over more than one card has them spread whole +// (expert parallelism); on one card there is nothing to spread. +func TestAModelOfExpertsSpreadsItsExpertsOverItsCards(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}} + moe := Shape{KVBytesPerToken: 94 * 4 * 256 * 2, KVHeads: 4, Params: 235_000_000_000, Experts: 128} + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 470 * gib, Quant: "bf16"}}, moe, nodes, 8192) + if len(opts) == 0 || opts[0].GPUsPerNode < 2 || !opts[0].ExpertParallel { + t.Fatalf("options %+v", opts) + } + small := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 30 * gib, Quant: "bf16"}}, + Shape{KVHeads: 4, Experts: 128}, nodes, 8192) + if len(small) == 0 || small[0].GPUsPerNode != 1 || small[0].ExpertParallel { + t.Errorf("one card: %+v", small) + } + dense := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}}, + Shape{KVHeads: 8}, nodes, 8192) + if len(dense) == 0 || dense[0].ExpertParallel { + t.Errorf("a dense model: %+v", dense) + } +} + +// Too big for two nodes, it is offered on three, not only on four. +func TestAModelTooBigForTwoNodesIsOfferedOnThree(t *testing.T) { + var nodes []PoolNode + for _, id := range []string{"a", "b", "c", "d"} { + nodes = append(nodes, PoolNode{ID: id, Class: "l40s-48g", GPUMemGB: 48, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"}) + } + // About 900 GB: more than two nodes' 16 cards hold, less than three's 24. + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 900 * gib, Quant: "bf16"}}, + Shape{KVHeads: 8}, nodes, 4096) + if len(opts) == 0 || opts[0].Nodes != 3 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" { + t.Fatalf("options %+v", labels(opts)) + } +} + +// Each way says what it needs part by part, and for another context. +func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}} + kvPerToken := int64(80 * 8 * 128 * 2 * 2) // a 70B: 320 KB a token + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}}, + Shape{KVBytesPerToken: kvPerToken, KVHeads: 8}, nodes, 8192) + o := opts[0] + if o.GPUsPerNode != 4 { + t.Fatalf("expected four cards: %+v", o) + } + sum := o.WeightsGB + o.ContextGB + o.EngineGB + if d := sum - o.NeedGB; d > 0.01 || d < -0.01 || o.WeightsGB != 35 { + t.Errorf("parts %.2f+%.2f+%.2f do not make %.2f", o.WeightsGB, o.ContextGB, o.EngineGB, o.NeedGB) + } + text := o.MemoryText() + for _, want := range []string{"GB a card", "weights 35.0", "context 0.6", "of 72 usable", "on 4 cards"} { + if !strings.Contains(text, want) { + t.Errorf("%q lacks %q", text, want) + } + } + // Sixteen times the context: the weights stay, the context grows, and + // it no longer fits. + long := o.WithContext(kvPerToken, 8192*16) + if long.WeightsGB != o.WeightsGB || long.ContextGB < o.ContextGB*15.9 { + t.Errorf("for a longer context: %+v", long) + } + if !o.Fits() { + t.Error("the planned way does not fit") + } + huge := o.WithContext(kvPerToken, 8192*256) + if huge.Fits() { + t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText()) + } +} diff --git a/internal/modelplan/size.go b/internal/modelplan/size.go index 4d1b0a6..18cc2f6 100644 --- a/internal/modelplan/size.go +++ b/internal/modelplan/size.go @@ -29,6 +29,10 @@ type Model struct { HeadDim int // MaxContext is the model's trained context length. MaxContext int + // Experts is a mixture-of-experts model's routed experts; 0 for a dense + // one. Its experts are spread over the cards (expert parallelism) rather + // than each sliced across them. + Experts int } // Precision is how many bytes one parameter weighs.