diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a6057f4..7f5a334 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -10516,6 +10516,20 @@ "bits_per_weight": { "type": "number", "description": "BitsPerWeight is the artifact's precision, 0 when unknown." + }, + "weights_gb": { + "type": "number", + "description": "What NeedGB is made of, a card's share (or, on the CPU, the node's RAM): the weights, the context's KV cache, and the engine's own." + }, + "context_gb": { + "type": "number" + }, + "engine_gb": { + "type": "number" + }, + "expert_parallel": { + "type": "boolean", + "description": "ExpertParallel spreads a mixture-of-experts model's experts over the cards, whole, rather than slicing each across them: vLLM's --enable-expert-parallel, for a model of experts on more than one card." } } }, @@ -10620,6 +10634,11 @@ "context": { "type": "integer" }, + "kv_bytes_per_token": { + "type": "integer", + "format": "int64", + "description": "KVBytesPerToken is one token of context, the whole model: what an option needs for another context is worked out from it. 0 is unknown." + }, "gated": { "type": "boolean" }, diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go index 6ee413d..429b8a8 100644 --- a/internal/cli/modelplan.go +++ b/internal/cli/modelplan.go @@ -291,14 +291,14 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string } fmt.Fprintln(out) w := tabTo(out) - fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW") + fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tA CARD\tOF\tALL CARDS\tNOW") for _, a := range p.Artifacts { name := a.Name if a.Shards > 1 && a.Format == "gguf" { name += fmt.Sprintf(" (%d parts)", a.Shards) } if a.Best == nil { - fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes)) + fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\t\n", name, bytesShort(a.Bytes)) continue } b := a.Best @@ -306,11 +306,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string if b.FreeNow { now = "free" } - holds := fmt.Sprintf("%.1f GB a card", b.NeedGB) + holds, all := fmt.Sprintf("%.1f GB", b.NeedGB), fmt.Sprintf("%.0f GB", b.TotalGB()) if b.CPU { - holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB) + holds, all = fmt.Sprintf("%.1f GB of RAM", b.NeedGB), "-" } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, all, now) } _ = w.Flush() rec, ok := p.Recommended() @@ -318,7 +318,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.") return errDoesNotFit } - fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label()) + fmt.Fprintf(out, "\nBest: %s on %s: %s.\n", rec.Artifact, rec.Label(), rec.MemoryText()) + if p.KVBytesPerToken > 0 { + fmt.Fprintf(out, "The context costs %.2f GB per 1000 tokens, the whole model (--context to plan another).\n", + modelplan.GB(p.KVBytesPerToken*1000)) + } if len(p.Options) > 1 { fmt.Fprintln(out, "\nEvery way, best first:") w = tabTo(out) @@ -327,7 +331,7 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string if o.FreeNow { now = "free" } - fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now) + fmt.Fprintf(w, " %s\t%s\t%s\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now) } _ = w.Flush() } diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index 337907f..fa2dea4 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -107,11 +107,7 @@ func optionLabel(o modelplan.Option) string { if o.FreeNow { now = sOK.Render("free") } - need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB) - if o.CPU { - need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB) - } - return fmt.Sprintf("%s · %s · %s", o.Label(), need, now) + return fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now) } // repoMenu offers a repository's files, each with the best way it runs here; @@ -197,8 +193,15 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact if p.Note != "" { note += "\n" + sWarn.Render("! ") + p.Note } - note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+ - "catalog, and a replica is started and asked to say hello.", p.Context) + note += fmt.Sprintf("\nMemory for a %d-token context, each way it runs:", p.Context) + for _, o := range opts { + note += "\n " + o.Label() + ": " + o.MemoryText() + } + if p.KVBytesPerToken > 0 { + note += fmt.Sprintf("\nThe context costs %.2f GB per 1000 tokens, the whole model; a longer one is checked\n"+ + "before it is served.", modelplan.GB(p.KVBytesPerToken*1000)) + } + note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello." ctxDefault := "4096" if a.Format == "safetensors" { ctxDefault = strconv.Itoa(p.Context) @@ -234,6 +237,10 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact req.File = a.Name } o := opts[slices.Index(where, v[4])] + // Asked for a context other than planned: does it still fit? + if sized := o.WithContext(p.KVBytesPerToken, ctxLen); !sized.Fits() { + return nil, fmt.Errorf("with a %d-token context it needs %s: a shorter context, or another way", ctxLen, sized.MemoryText()) + } if o.CPU { req.CPU = true } else { diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index eb773b8..d5363b4 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -461,8 +461,17 @@ func (s *Server) run(j *Job) { for _, o := range layout { where = append(where, o.Label()) } - s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s", - set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "))) + // What the first way needs of each card, for the context asked for. + need := "" + if first := layout[0]; first.NeedGB > 0 { + tokens := req.ContextLength + if req.Format != "safetensors" && req.CtxPool > 0 { + tokens = req.CtxPool + } + need = "; needs " + first.WithContext(plan.KVBytesPerToken, tokens).MemoryText() + } + s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s%s", + set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "), need)) // 2. The mirror: fetch, and check what arrived against what was published. source, err := s.mirror(ctx, j, repo, set, name) @@ -622,6 +631,11 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode } p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)} + if o.ExpertParallel { + // The experts whole on each card rather than each sliced + // across them: less traffic between the cards for every token. + p.Args = append(p.Args, "--enable-expert-parallel") + } } out = append(out, p) } diff --git a/internal/control/modelserve/plan.go b/internal/control/modelserve/plan.go index aef850b..38fa16d 100644 --- a/internal/control/modelserve/plan.go +++ b/internal/control/modelserve/plan.go @@ -42,7 +42,10 @@ type Plan struct { License string `json:"license,omitempty"` Params int64 `json:"params,omitempty"` Context int `json:"context"` - Gated bool `json:"gated,omitempty"` + // KVBytesPerToken is one token of context, the whole model: what an + // option needs for another context is worked out from it. 0 is unknown. + KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"` + Gated bool `json:"gated,omitempty"` // Servable is false when the nodes' engine cannot run the model at all, // and Reason says why. Servable bool `json:"servable"` @@ -144,6 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, if err == nil { shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16) shape.KVHeads = m.KVHeads + shape.Experts = m.Experts if p.Params == 0 { p.Params = m.Params } @@ -157,6 +161,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, p.Note = "the model's shape was not readable, so the context's memory is not counted" } shape.Params = p.Params + p.KVBytesPerToken = shape.KVBytesPerToken p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context) if !p.Servable { p.Options = nil diff --git a/internal/modelplan/fit.go b/internal/modelplan/fit.go index 3dacb53..7175d2c 100644 --- a/internal/modelplan/fit.go +++ b/internal/modelplan/fit.go @@ -155,11 +155,11 @@ func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit { if nodes > 1 && h.Fabric == "" { continue } - // The KV heads have to divide too, and they are far fewer than the - // attention heads under grouped-query attention: a model with 8 KV - // heads cannot be split sixteen ways. - if m.KVHeads > 0 && width > m.KVHeads { - break + // The KV heads are split among the cards, or past their number each + // card holds a copy of one: the width divides them or they divide + // it. A model with 8 KV heads splits 2, 4, 8 or 16 ways, not 12. + if m.KVHeads > 0 && m.KVHeads%width != 0 && width%m.KVHeads != 0 { + continue } per := GB(d.PerGPU(width)) diff --git a/internal/modelplan/fit_test.go b/internal/modelplan/fit_test.go index 364ffe4..d7eda45 100644 --- a/internal/modelplan/fit_test.go +++ b/internal/modelplan/fit_test.go @@ -89,13 +89,14 @@ func TestATensorGroupNeverSpansFabrics(t *testing.T) { } } -func TestTensorWidthCannotExceedTheKVHeads(t *testing.T) { - // Grouped-query attention leaves far fewer KV heads than attention heads, - // and an engine refuses a width that cannot divide them. Offering tp=16 for - // a model with 8 KV heads is a plan that fails at launch. +func TestTensorWidthFitsTheKVHeads(t *testing.T) { + // Grouped-query attention leaves far fewer KV heads than attention heads. + // An engine splits them among the cards, or past their number gives each + // card a copy of one: so a width that neither divides them nor is a + // multiple of them fails at launch, and is not offered. for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) { - if f.TP > llama70B.KVHeads { - t.Errorf("tp=%d exceeds the model's %d KV heads", f.TP, llama70B.KVHeads) + if llama70B.KVHeads%f.TP != 0 && f.TP%llama70B.KVHeads != 0 { + t.Errorf("tp=%d does not fit the model's %d KV heads", f.TP, llama70B.KVHeads) } } } diff --git a/internal/modelplan/huggingface.go b/internal/modelplan/huggingface.go index 9c6d381..0b7a94d 100644 --- a/internal/modelplan/huggingface.go +++ b/internal/modelplan/huggingface.go @@ -47,9 +47,20 @@ type hfConfig struct { HeadDim int `json:"head_dim"` MaxPositions int `json:"max_position_embeddings"` ModelType string `json:"model_type"` + // The routed experts of a mixture-of-experts model, by whichever name + // its family gives them: Qwen's and Mixtral's, DeepSeek's, Llama 4's. + NumExperts int `json:"num_experts"` + NRoutedExperts int `json:"n_routed_experts"` + NumLocalExperts int `json:"num_local_experts"` + // TextConfig is where a model that also reads pictures keeps its + // language model's figures (Llama 4, Qwen3-VL, Gemma 3). + TextConfig *hfConfig `json:"text_config"` } func (c hfConfig) toModel(repo string) (Model, error) { + if c.NumHiddenLayers == 0 && c.TextConfig != nil { + c = *c.TextConfig + } if c.NumHiddenLayers == 0 { return Model{}, fmt.Errorf("%s: config.json has no num_hidden_layers; is it a text model?", repo) } @@ -57,6 +68,7 @@ func (c hfConfig) toModel(repo string) (Model, error) { Name: repo[strings.LastIndex(repo, "/")+1:], Layers: c.NumHiddenLayers, MaxContext: c.MaxPositions, + Experts: max(c.NumExperts, c.NRoutedExperts, c.NumLocalExperts), } // Grouped-query attention: when the KV head count is absent the model has diff --git a/internal/modelplan/huggingface_test.go b/internal/modelplan/huggingface_test.go index 94fc5c2..d4c879d 100644 --- a/internal/modelplan/huggingface_test.go +++ b/internal/modelplan/huggingface_test.go @@ -75,3 +75,15 @@ func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) { t.Errorf("Name = %q", m.Name) } } + +func TestAModelThatReadsPicturesIsSizedByItsLanguageModel(t *testing.T) { + m, err := hfConfig{TextConfig: &hfConfig{NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8, + HiddenSize: 5120, NumLocalExperts: 16}}.toModel("meta-llama/Llama-4-Scout-17B-16E-Instruct") + if err != nil || m.Layers != 48 || m.KVHeads != 8 || m.Experts != 16 { + t.Errorf("%+v %v", m, err) + } + ds, _ := hfConfig{NumHiddenLayers: 61, NumAttentionHeads: 128, HiddenSize: 7168, NRoutedExperts: 256}.toModel("deepseek-ai/DeepSeek-V3") + if ds.Experts != 256 { + t.Errorf("DeepSeek's experts: %d", ds.Experts) + } +} diff --git a/internal/modelplan/options.go b/internal/modelplan/options.go index bb1217d..deb295e 100644 --- a/internal/modelplan/options.go +++ b/internal/modelplan/options.go @@ -33,6 +33,8 @@ type Shape struct { KVHeads int // Params is the parameter count, for bits per weight; 0 is unknown. Params int64 + // Experts is a mixture-of-experts model's routed experts; 0 for dense. + Experts int } // PoolNode is one ready node, as a plan sees it. @@ -96,11 +98,71 @@ type Option struct { FreeNow bool `json:"free_now"` // BitsPerWeight is the artifact's precision, 0 when unknown. BitsPerWeight float64 `json:"bits_per_weight,omitempty"` + // What NeedGB is made of, a card's share (or, on the CPU, the node's + // RAM): the weights, the context's KV cache, and the engine's own. + WeightsGB float64 `json:"weights_gb,omitempty"` + ContextGB float64 `json:"context_gb,omitempty"` + EngineGB float64 `json:"engine_gb,omitempty"` + // ExpertParallel spreads a mixture-of-experts model's experts over the + // cards, whole, rather than slicing each across them: vLLM's + // --enable-expert-parallel, for a model of experts on more than one card. + ExpertParallel bool `json:"expert_parallel,omitempty"` } // GPUs is every card one replica takes. func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) } +// TotalGB is what one replica takes of every card together, or of the +// node's RAM on the CPU. +func (o Option) TotalGB() float64 { + if o.CPU { + return o.NeedGB + } + return o.NeedGB * float64(o.GPUs()) +} + +// Fits is whether a card, or the node, holds what it needs. +func (o Option) Fits() bool { return o.NeedGB <= o.HaveGB } + +// WithContext is the option for another context: kvPerToken a token's KV +// cache, the whole model; the weights and the engine's share stay. +func (o Option) WithContext(kvPerToken int64, tokens int) Option { + if kvPerToken <= 0 || tokens <= 0 { + return o + } + ctx := GB(int64(tokens) * kvPerToken) + if !o.CPU { + ctx /= float64(max(o.GPUs(), 1)) + } + o.NeedGB += ctx - o.ContextGB + o.ContextGB = ctx + return o +} + +// MemoryText says what it needs, part by part. +func (o Option) MemoryText() string { + parts := fmt.Sprintf("weights %.1f + context %.1f + engine %.1f", o.WeightsGB, o.ContextGB, o.EngineGB) + if o.WeightsGB == 0 && o.ContextGB == 0 { + parts = "" + } + if o.CPU { + s := fmt.Sprintf("%.1f GB of RAM", o.NeedGB) + if parts != "" { + s += " (" + parts + ")" + } + return s + fmt.Sprintf(" of %.0f usable", o.HaveGB) + } + s := fmt.Sprintf("%.1f GB a card", o.NeedGB) + if parts != "" { + s += " (" + parts + ")" + } + s += fmt.Sprintf(" of %.0f usable", o.HaveGB) + if o.GPUs() > 1 { + s += fmt.Sprintf("; %.0f GB on %d cards", o.TotalGB(), o.GPUs()) + } + return s +} + // Label says where it runs, in a few words. func (o Option) Label() string { if o.CPU { @@ -151,7 +213,8 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { // The CPU: the whole replica in a node's RAM. total := a.Bytes + kv + ggufOverhead(a.Bytes) cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true, - NeedGB: GB(total), BitsPerWeight: bpw} + NeedGB: GB(total), BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes), ContextGB: GB(kv), EngineGB: GB(ggufOverhead(a.Bytes))} for _, n := range nodes { have := float64(n.HostMemGB) * hostShare if n.HostMemGB <= 0 || GB(total) > have { @@ -175,7 +238,9 @@ func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option { continue } o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class, - GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw} + GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes / int64(width)), ContextGB: GB(kv / int64(width)), + EngineGB: GB(ggufOverhead(a.Bytes / int64(width)))} for _, fab := range c.fabrics { for _, n := range fab.nodes { if n.GPUs >= width { @@ -198,13 +263,20 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) perCard := func(width int) int64 { return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width)) } - // Tensor parallel within a node must divide the KV heads. - divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) } + // Tensor parallel splits the KV heads among the cards, or past their + // number gives each card a copy of one: so the width divides the heads, + // or the heads divide the width. Qwen3-235B has four and runs on eight. + divides := func(w int) bool { + return sh.KVHeads == 0 || sh.KVHeads%w == 0 || w%sh.KVHeads == 0 + } for _, c := range classes(nodes) { budget := float64(c.mem) * gpuShare opt := func(perNode, n int, fabric string) Option { + cards := int64(perNode * n) return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric, - GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw} + GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw, + WeightsGB: GB(a.Bytes / cards), ContextGB: GB(kv / cards), EngineGB: GB(overheadBytes(a.Bytes / cards)), + ExpertParallel: sh.Experts > 1 && perNode*n > 1} } // One node: the narrowest power of two that holds it. single := false @@ -243,7 +315,10 @@ func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) if fab.name == "" { continue } - for n := 2; n <= len(fab.nodes); n *= 2 { + // Any number of nodes: pipeline stages need not be a power of + // two, and a third node is what a model just too big for two + // needs. + for n := 2; n <= len(fab.nodes); n++ { if GB(perCard(w*n)) > budget { continue } diff --git a/internal/modelplan/options_test.go b/internal/modelplan/options_test.go index 476bbe4..768d594 100644 --- a/internal/modelplan/options_test.go +++ b/internal/modelplan/options_test.go @@ -123,3 +123,73 @@ func TestOnTheCPUAMiddlingQuantisationIsBestAndOnACardEightBits(t *testing.T) { t.Errorf("GPU: %s first", o[0].Artifact) } } + +// A model of experts spread over more than one card has them spread whole +// (expert parallelism); on one card there is nothing to spread. +func TestAModelOfExpertsSpreadsItsExpertsOverItsCards(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}} + moe := Shape{KVBytesPerToken: 94 * 4 * 256 * 2, KVHeads: 4, Params: 235_000_000_000, Experts: 128} + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 470 * gib, Quant: "bf16"}}, moe, nodes, 8192) + if len(opts) == 0 || opts[0].GPUsPerNode < 2 || !opts[0].ExpertParallel { + t.Fatalf("options %+v", opts) + } + small := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 30 * gib, Quant: "bf16"}}, + Shape{KVHeads: 4, Experts: 128}, nodes, 8192) + if len(small) == 0 || small[0].GPUsPerNode != 1 || small[0].ExpertParallel { + t.Errorf("one card: %+v", small) + } + dense := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}}, + Shape{KVHeads: 8}, nodes, 8192) + if len(dense) == 0 || dense[0].ExpertParallel { + t.Errorf("a dense model: %+v", dense) + } +} + +// Too big for two nodes, it is offered on three, not only on four. +func TestAModelTooBigForTwoNodesIsOfferedOnThree(t *testing.T) { + var nodes []PoolNode + for _, id := range []string{"a", "b", "c", "d"} { + nodes = append(nodes, PoolNode{ID: id, Class: "l40s-48g", GPUMemGB: 48, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"}) + } + // About 900 GB: more than two nodes' 16 cards hold, less than three's 24. + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 900 * gib, Quant: "bf16"}}, + Shape{KVHeads: 8}, nodes, 4096) + if len(opts) == 0 || opts[0].Nodes != 3 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" { + t.Fatalf("options %+v", labels(opts)) + } +} + +// Each way says what it needs part by part, and for another context. +func TestAnOptionSaysWhatItNeedsOfEachCardAndOfAll(t *testing.T) { + nodes := []PoolNode{{ID: "a", Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8}} + kvPerToken := int64(80 * 8 * 128 * 2 * 2) // a 70B: 320 KB a token + opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib, Quant: "bf16"}}, + Shape{KVBytesPerToken: kvPerToken, KVHeads: 8}, nodes, 8192) + o := opts[0] + if o.GPUsPerNode != 4 { + t.Fatalf("expected four cards: %+v", o) + } + sum := o.WeightsGB + o.ContextGB + o.EngineGB + if d := sum - o.NeedGB; d > 0.01 || d < -0.01 || o.WeightsGB != 35 { + t.Errorf("parts %.2f+%.2f+%.2f do not make %.2f", o.WeightsGB, o.ContextGB, o.EngineGB, o.NeedGB) + } + text := o.MemoryText() + for _, want := range []string{"GB a card", "weights 35.0", "context 0.6", "of 72 usable", "on 4 cards"} { + if !strings.Contains(text, want) { + t.Errorf("%q lacks %q", text, want) + } + } + // Sixteen times the context: the weights stay, the context grows, and + // it no longer fits. + long := o.WithContext(kvPerToken, 8192*16) + if long.WeightsGB != o.WeightsGB || long.ContextGB < o.ContextGB*15.9 { + t.Errorf("for a longer context: %+v", long) + } + if !o.Fits() { + t.Error("the planned way does not fit") + } + huge := o.WithContext(kvPerToken, 8192*256) + if huge.Fits() { + t.Errorf("a 2M-token context fits four cards: %s", huge.MemoryText()) + } +} diff --git a/internal/modelplan/size.go b/internal/modelplan/size.go index 4d1b0a6..18cc2f6 100644 --- a/internal/modelplan/size.go +++ b/internal/modelplan/size.go @@ -29,6 +29,10 @@ type Model struct { HeadDim int // MaxContext is the model's trained context length. MaxContext int + // Experts is a mixture-of-experts model's routed experts; 0 for a dense + // one. Its experts are spread over the cards (expert parallelism) rather + // than each sliced across them. + Experts int } // Precision is how many bytes one parameter weighs.