From 300e2980538218d55e0d0e3729fbb7a4ec61cc3a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 17 Sep 2026 07:46:53 +0200 Subject: [PATCH] Work out whether a model fits the cluster, and how to split it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct The arithmetic this replaces is the last thing between having GPUs and serving a model on them. Choosing tp, gpu_class and a context that will actually fit means knowing the layer count, the KV head count, what the weights weigh at a chosen quantisation and how much of a card the runtime keeps — and getting it wrong is not an error message, it is an out-of-memory twenty minutes into a load. I computed those by hand for every model on this cluster. Both halves are read rather than retyped: the architecture from the model's own config.json and parameter count, the hardware from the running fleet. Attention heads in place of KV heads is the classic slip and is worth a factor of five on a model with grouped-query attention, so it is not a number anyone should be entering. Every candidate is listed, not one answer. With cards to spare you want the full-precision fit; with one card free you want to know a quantised one exists. Choosing between those is not the planner's job. The tests are calibrated against files actually downloaded here — llama.cpp reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the model reproduces both — and they caught two errors in my own arithmetic. The overhead was charged per rank from the whole model's weights rather than each rank's share, and it reserved a tenth of a card that usableFraction had already reserved. Between them, Llama-3.3-70B at bf16 across two 80 GB cards came out as not fitting, which is a configuration people run every day. Slack reserved in two places, each unaware of the other, reads as caution and behaves as a bug. Pools are grouped by class and fabric together, because a replica may never span two fabrics: eight free cards split across two of them are not eight cards for a model that needs eight, and merging them would plan something that starts, passes its health checks and hangs on its first all-reduce. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/model.go | 2 +- internal/cli/modelplan.go | 227 +++++++++++++++++++++++++ internal/cli/status_test.go | 84 +++++++++ internal/modelplan/fit.go | 167 ++++++++++++++++++ internal/modelplan/fit_test.go | 150 ++++++++++++++++ internal/modelplan/huggingface.go | 135 +++++++++++++++ internal/modelplan/huggingface_test.go | 77 +++++++++ internal/modelplan/size.go | 43 +++-- internal/modelplan/size_test.go | 160 +++++++++++++++++ 9 files changed, 1029 insertions(+), 16 deletions(-) create mode 100644 internal/cli/modelplan.go create mode 100644 internal/modelplan/fit.go create mode 100644 internal/modelplan/fit_test.go create mode 100644 internal/modelplan/huggingface.go create mode 100644 internal/modelplan/huggingface_test.go create mode 100644 internal/modelplan/size_test.go diff --git a/internal/cli/model.go b/internal/cli/model.go index 374d1bd7..0a7e00c3 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -25,7 +25,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command { Use: "model", Short: "Add or remove models while the cluster is running", } - c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client)) + c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client)) return c } diff --git a/internal/cli/modelplan.go b/internal/cli/modelplan.go new file mode 100644 index 00000000..3caedbad --- /dev/null +++ b/internal/cli/modelplan.go @@ -0,0 +1,227 @@ +package cli + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/modelplan" +) + +// newModelPlanCmd answers whether a model fits this cluster, and how. +// +// The arithmetic it replaces is the last thing standing between having GPUs and +// serving a model on them. Deciding tp, gpu_class and a context that will +// actually fit means knowing the model's layer count, its KV head count, what +// its weights weigh at a chosen quantisation, and how much of a card the +// runtime keeps — and getting it wrong is not an error message, it is an +// out-of-memory twenty minutes into a load. +// +// It reads the model from its own repository and the hardware from the running +// fleet, so neither half is a number somebody retyped. +func newModelPlanCmd(client func() *ctlClient) *cobra.Command { + var ( + context int + params int64 + precs []string + ) + c := &cobra.Command{ + Use: "plan ", + Short: "Work out whether a model fits this cluster, and how it should be split", + Long: "Reads the model's architecture from its repository and the hardware from\n" + + "the running fleet, then prints every way it could run — fewest GPUs\n" + + "first — and a catalog entry for the best one.\n\n" + + "Every candidate is listed rather than one answer, because the choice is\n" + + "not the planner's to make: with cards to spare you want the\n" + + "full-precision fit, and with one card free you want to know a quantised\n" + + "one exists.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + repo := args[0] + m, err := modelplan.FetchModel(cmd.Context(), repo) + if err != nil { + return err + } + if params > 0 { + m.Params = params + } + if m.Params == 0 { + return fmt.Errorf("%s does not publish a parameter count; pass --params", repo) + } + + var chosen []modelplan.Precision + for _, name := range precs { + p, err := modelplan.PrecisionByName(name) + if err != nil { + return err + } + chosen = append(chosen, p) + } + + var nodes []adminapi.NodeView + if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil { + return err + } + hw := hardwareFrom(nodes) + if len(hw) == 0 { + return fmt.Errorf("no node is ready, so there is nothing to plan against") + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%s: %.1fB parameters, %d layers, %d KV heads, trained to %d tokens\n", + m.Name, float64(m.Params)/1e9, m.Layers, m.KVHeads, m.MaxContext) + + want := context + if want <= 0 { + want = m.MaxContext + } + fmt.Fprintf(out, "Planned for a %d-token pool. %s a token of context.\n\n", + want, humanBytes(m.KVBytesPerToken(modelplan.BF16))) + + fits := modelplan.Fits(m, hw, want, chosen) + if len(fits) == 0 { + fmt.Fprintln(out, "It does not fit this cluster at any offered precision.") + fmt.Fprintln(out, "\nWhat would change that:") + fmt.Fprintln(out, " - a smaller context pool, with --context") + fmt.Fprintln(out, " - a heavier quantisation, with --precision q4_k_m") + fmt.Fprintln(out, " - more cards free; llmgw ctl status shows what is in use") + return errDoesNotFit + } + + w := tab() + fmt.Fprintln(w, " GPUS\tPRECISION\tPER CARD\tFREE\tWHERE") + for _, f := range fits { + where := f.Class + if f.CPUOnly { + where = "cpu" + } else if f.Nodes > 1 { + where = fmt.Sprintf("%s, %d nodes on %s", f.Class, f.Nodes, f.Fabric) + } + fmt.Fprintf(w, " %d\t%s\t%.1f GB\t%.1f GB\t%s\n", + f.TP, f.Precision.Name, f.PerGPUGB, f.Headroom, where) + } + _ = w.Flush() + + fmt.Fprintf(out, "\nA catalog entry for the first of those:\n\n%s\n", + catalogEntry(m, fits[0], repo, want)) + fmt.Fprintln(out, "Save it and add it with: llmgw ctl model add -f ") + return nil + }, + } + c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)") + c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one") + c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)") + return c +} + +// errDoesNotFit gives a script something to test without parsing prose. +var errDoesNotFit = fmt.Errorf("the model does not fit this cluster") + +// hardwareFrom turns the fleet into pools a replica could actually occupy. +// +// Grouped by class and fabric together, because a replica may never span two +// fabrics: eight free cards split across two fabrics are not eight cards for a +// model that needs eight, and treating them as one pool would plan something +// that hangs on its first all-reduce. +func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware { + type key struct{ class, fabric string } + pools := map[key]*modelplan.Hardware{} + + for _, n := range nodes { + if n.State != "ready" { + continue + } + fabric := "" + if len(n.Fabrics) > 0 { + fabric = n.Fabrics[0] + } + class := "" + if len(n.Classes) > 0 { + class = n.Classes[0] + } + k := key{class, fabric} + if pools[k] == nil { + pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric} + } + p := pools[k] + p.Nodes++ + // The widest a single node could contribute, not the total: a tensor + // group takes the same number of cards from every node it spans. + if n.GPUFree > p.GPUsPerNode { + p.GPUsPerNode = n.GPUFree + } + if n.MemoryGB > p.HostMemGB { + p.HostMemGB = n.MemoryGB + } + // Card memory from the class label, which carries it by construction: + // "h100-80g" is eighty gigabytes and says so. + if class != "" && p.GPUMemGB == 0 { + p.GPUMemGB = memFromClass(class) + } + } + + out := make([]modelplan.Hardware, 0, len(pools)) + for _, p := range pools { + if p.Class != "" && (p.GPUMemGB == 0 || p.GPUsPerNode == 0) { + continue // a GPU pool with no free cards is not a pool + } + out = append(out, *p) + } + return out +} + +// memFromClass reads the card size out of a class label such as "h100-80g". +func memFromClass(class string) int { + i := strings.LastIndex(class, "-") + if i < 0 || !strings.HasSuffix(class, "g") { + return 0 + } + var gb int + if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil { + return 0 + } + return gb +} + +// catalogEntry renders a fit as the YAML that would serve it. +func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string { + var b strings.Builder + name := strings.ToLower(m.Name) + fmt.Fprintf(&b, "name: %s\n", name) + fmt.Fprintf(&b, "source: %s\n", source) + fmt.Fprintf(&b, "routing_profile: chat\n") + fmt.Fprintf(&b, "context_length: %d\n", context) + fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n") + fmt.Fprintf(&b, "profiles:\n") + + switch { + case f.CPUOnly: + fmt.Fprintf(&b, " - name: cpu\n engine: llamacpp\n") + fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2) + default: + fmt.Fprintf(&b, " - name: tp%d\n engine: vllm\n", f.TP) + fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP) + if f.Nodes > 1 { + fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode) + } + } + fmt.Fprintf(&b, " priority: 100\n weight: 1\n") + // Measured rather than guessed is the rule everywhere else in this system, + // and a planner should say so rather than pretend its estimate is one. + fmt.Fprintf(&b, " startup_timeout: 10m\n") + b.WriteString("\n# weight should come from `llmgw ctl bench` once it is running;\n") + b.WriteString("# 1 is a placeholder, not a measurement.\n") + return b.String() +} + +// humanBytes renders a per-token cache size, which is always small. +func humanBytes(n int64) string { + switch { + case n >= 1<<20: + return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) + default: + return fmt.Sprintf("%.0f KiB", float64(n)/1024) + } +} diff --git a/internal/cli/status_test.go b/internal/cli/status_test.go index 0e4f007f..7bf538d9 100644 --- a/internal/cli/status_test.go +++ b/internal/cli/status_test.go @@ -59,3 +59,87 @@ func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) { }) } } + +func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) { + // A replica may never span two fabrics, so eight free cards split across + // two of them are not eight cards for a model that needs eight. Merging + // them would plan something that starts, passes health checks and hangs on + // its first all-reduce. + nodes := []adminapi.NodeView{ + {ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4}, + {ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4}, + {ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4}, + } + hw := hardwareFrom(nodes) + if len(hw) != 2 { + t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw) + } + for _, p := range hw { + switch p.Fabric { + case "ib-a": + if p.Nodes != 2 { + t.Errorf("ib-a has %d nodes, want 2", p.Nodes) + } + case "ib-b": + if p.Nodes != 1 { + t.Errorf("ib-b has %d nodes, want 1", p.Nodes) + } + default: + t.Errorf("unexpected fabric %q", p.Fabric) + } + } +} + +func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) { + // Planning against a node that is draining or unreachable produces a fit + // that cannot be placed, which is worse than saying nothing fits. + nodes := []adminapi.NodeView{ + {ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8}, + {ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8}, + {ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8}, + } + hw := hardwareFrom(nodes) + if len(hw) != 1 || hw[0].Nodes != 1 { + t.Fatalf("expected only the ready node, got %+v", hw) + } +} + +func TestCardMemoryComesFromTheClassLabel(t *testing.T) { + // The label carries it by construction — "h100-80g" is eighty gigabytes and + // says so — which is why the planner needs no table of card models. + for label, want := range map[string]int{ + "h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24, + "nonsense": 0, "": 0, "h100-": 0, + } { + if got := memFromClass(label); got != want { + t.Errorf("memFromClass(%q) = %d, want %d", label, got, want) + } + } +} + +func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) { + // How this cluster began: a node with no cards is not an empty pool, it is + // a pool measured in RAM. + hw := hardwareFrom([]adminapi.NodeView{ + {ID: "cpu1", State: "ready", MemoryGB: 31}, + }) + if len(hw) != 1 { + t.Fatalf("the CPU node produced %d pools", len(hw)) + } + if hw[0].Class != "" || hw[0].HostMemGB != 31 { + t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0]) + } +} + +func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) { + // Fully allocated hardware is not capacity, and offering it would plan a + // replica that cannot be placed. + hw := hardwareFrom([]adminapi.NodeView{ + {ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8}, + }) + for _, p := range hw { + if p.Class != "" { + t.Errorf("a node with no free cards became a pool: %+v", p) + } + } +} diff --git a/internal/modelplan/fit.go b/internal/modelplan/fit.go new file mode 100644 index 00000000..0e410a62 --- /dev/null +++ b/internal/modelplan/fit.go @@ -0,0 +1,167 @@ +package modelplan + +import ( + "fmt" + "sort" +) + +// Hardware is what one kind of node offers, as the fleet reports it. +type Hardware struct { + // Class is the GPU class, e.g. "h100-80g". Empty means a CPU-only node. + Class string + // GPUMemGB is one card's memory. + GPUMemGB int + // GPUsPerNode is how many cards a node of this kind has free. + GPUsPerNode int + // Nodes is how many such nodes are available. + Nodes int + // Fabric is what they are attached to, empty when they are not. A replica + // may never span two fabrics, so nodes on different ones are separate + // pools however identical their cards. + Fabric string + // HostMemGB is the machine's RAM, which is what a CPU profile needs. + HostMemGB int +} + +// Fit is one way a model could run. +type Fit struct { + Precision Precision + Class string + TP int + Nodes int + GPUsPerNode int + Fabric string + // Context is the token pool this fit was checked against. + Context int + // PerGPUGB is what each card must hold, and Headroom what is left of it. + PerGPUGB float64 + Headroom float64 + // CPUOnly marks a fit that needs no GPU at all. + CPUOnly bool +} + +// usableFraction is how much of a card a plan may assume it can have. +// +// Not all of it. A framework's allocator fragments, the driver reserves some, +// and a card planned to ninety-nine percent full is a card that fails to load +// on a bad day rather than a good one. vLLM's own default GPU utilisation is +// 0.90 and this matches it deliberately: a plan that disagrees with the engine +// it is planning for is worse than no plan. +const usableFraction = 0.90 + +// Fits returns every way this model could run on the given hardware, best +// first — fewest GPUs, then fewest nodes, then highest precision. +// +// Every candidate, not one answer, because the choice between them is not the +// planner's to make: a team with spare cards wants the full-precision fit, and +// a team with one card free wants to know a quantised one exists. Returning +// only the "best" would hide the option someone actually wants. +func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit { + if context <= 0 { + context = m.MaxContext + } + if len(precisions) == 0 { + precisions = []Precision{BF16, FP8, Q8GGUF, Q4GGUF} + } + + var out []Fit + for _, p := range precisions { + d := m.Needs(p, context) + for _, h := range hw { + out = append(out, fitsOn(m, h, d, p, context)...) + } + } + + sort.SliceStable(out, func(i, j int) bool { + a, b := out[i], out[j] + if ga, gb := a.TP, b.TP; ga != gb { + return ga < gb + } + if a.Nodes != b.Nodes { + return a.Nodes < b.Nodes + } + // Higher precision first among equals: the same hardware serving a + // better model is strictly the better answer. + return a.Precision.BytesPerParam > b.Precision.BytesPerParam + }) + return out +} + +func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit { + if h.Class == "" { + return cpuFit(h, d, p, context) + } + + var out []Fit + budget := float64(h.GPUMemGB) * usableFraction + + // Powers of two only. Tensor parallelism divides attention heads across + // ranks, and an engine that cannot divide them evenly refuses the profile — + // so offering tp=3 would produce a plan that fails at launch. + for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 { + nodes := 1 + perNode := width + if width > h.GPUsPerNode { + perNode = h.GPUsPerNode + nodes = width / h.GPUsPerNode + } + if nodes > h.Nodes { + break + } + // A replica spanning nodes needs a fabric. Without one every node is + // its own island, and a multi-node replica would hang on its first + // all-reduce rather than fail to start. + if nodes > 1 && h.Fabric == "" { + continue + } + // The KV heads have to divide too, and they are far fewer than the + // attention heads under grouped-query attention: a model with 8 KV + // heads cannot be split sixteen ways. + if m.KVHeads > 0 && width > m.KVHeads { + break + } + + per := GB(d.PerGPU(width)) + if per > budget { + continue + } + out = append(out, Fit{ + Precision: p, Class: h.Class, TP: width, Nodes: nodes, + GPUsPerNode: perNode, Fabric: h.Fabric, Context: context, + PerGPUGB: per, Headroom: budget - per, + }) + } + return out +} + +// cpuFit checks a node with no cards, where the whole working set lives in RAM. +func cpuFit(h Hardware, d Demand, p Precision, context int) []Fit { + if h.HostMemGB <= 0 { + return nil + } + // Less of the machine than of a card: the operating system, the page cache + // and everything else on the node want their share, and a CPU node is + // rarely dedicated the way a GPU node is. + budget := float64(h.HostMemGB) * 0.75 + need := GB(d.Total()) + if need > budget { + return nil + } + return []Fit{{ + Precision: p, TP: 1, Nodes: 1, Context: context, + PerGPUGB: need, Headroom: budget - need, CPUOnly: true, + }} +} + +// Describe renders a fit the way someone would say it out loud. +func (f Fit) Describe() string { + if f.CPUOnly { + return fmt.Sprintf("CPU, %s, %.1f GB of RAM", f.Precision.Name, f.PerGPUGB) + } + where := fmt.Sprintf("%d×%s", f.TP, f.Class) + if f.Nodes > 1 { + where = fmt.Sprintf("%d nodes × %d×%s on %s", f.Nodes, f.GPUsPerNode, f.Class, f.Fabric) + } + return fmt.Sprintf("%s, %s, %.1f GB a card (%.1f free)", + where, f.Precision.Name, f.PerGPUGB, f.Headroom) +} diff --git a/internal/modelplan/fit_test.go b/internal/modelplan/fit_test.go new file mode 100644 index 00000000..01c93a38 --- /dev/null +++ b/internal/modelplan/fit_test.go @@ -0,0 +1,150 @@ +package modelplan + +import ( + "strings" + "testing" +) + +func h100Node(nodes, gpus int, fabric string) Hardware { + return Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: gpus, Nodes: nodes, Fabric: fabric} +} + +func TestASmallModelFitsOneCard(t *testing.T) { + fits := Fits(coder3B, []Hardware{h100Node(4, 8, "ib-a")}, 32768, []Precision{BF16}) + if len(fits) == 0 { + t.Fatal("a 3B was reported as not fitting an H100") + } + // Best first, and the best answer for a small model is one card. + if fits[0].TP != 1 { + t.Errorf("best fit uses %d GPUs; a 3B needs one", fits[0].TP) + } +} + +func TestA70BNeedsTwoCardsAtFullPrecision(t *testing.T) { + // The configuration people run every day, and the one an earlier overhead + // model wrongly called impossible by reserving slack twice. + fits := Fits(llama70B, []Hardware{h100Node(4, 8, "ib-a")}, 8192, []Precision{BF16}) + if len(fits) == 0 { + t.Fatal("70B bf16 reported as not fitting eight H100s") + } + if fits[0].TP != 2 { + t.Errorf("best fit is tp=%d; 70B at bf16 on two 80 GB cards is routine", fits[0].TP) + } + for _, f := range fits { + if f.TP == 1 { + t.Errorf("70B bf16 reported as fitting a single card: %s", f.Describe()) + } + } +} + +func TestContextChangesHowManyCardsAModelNeeds(t *testing.T) { + // The fact that makes this worth computing rather than looking up. The same + // 70B at the same precision needs two cards for an 8k pool and four for + // 32k, because the KV cache is what grows — and "70B needs two H100s" is + // repeated everywhere without the context it was true for. + hw := []Hardware{h100Node(4, 8, "ib-a")} + short := Fits(llama70B, hw, 8192, []Precision{BF16}) + long := Fits(llama70B, hw, 32768, []Precision{BF16}) + if len(short) == 0 || len(long) == 0 { + t.Fatal("expected fits at both context sizes") + } + if long[0].TP <= short[0].TP { + t.Errorf("tp at 32k is %d and at 8k is %d; a larger context should need at least as much", + long[0].TP, short[0].TP) + } +} + +func TestQuantisationOpensUpSmallerHardware(t *testing.T) { + // The answer someone with one spare card needs: not "no", but "not at this + // precision". + one := []Hardware{h100Node(1, 1, "")} + if full := Fits(llama70B, one, 8192, []Precision{BF16}); len(full) != 0 { + t.Errorf("70B bf16 fitted a single card: %s", full[0].Describe()) + } + if quant := Fits(llama70B, one, 8192, []Precision{Q4GGUF}); len(quant) == 0 { + t.Error("70B at q4 did not fit a single 80 GB card, which it should") + } +} + +func TestATensorGroupNeverSpansFabrics(t *testing.T) { + // Two nodes with no fabric are two islands. A multi-node replica placed + // across them would start, pass health checks and hang on its first + // all-reduce, which is worse than refusing to plan it. + noFabric := Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: 2, Nodes: 4} + for _, f := range Fits(llama70B, []Hardware{noFabric}, 8192, []Precision{BF16}) { + if f.Nodes > 1 { + t.Errorf("planned %d nodes with no fabric: %s", f.Nodes, f.Describe()) + } + } + withFabric := noFabric + withFabric.Fabric = "ib-a" + var spanning bool + for _, f := range Fits(llama70B, []Hardware{withFabric}, 8192, []Precision{BF16}) { + if f.Nodes > 1 { + spanning = true + } + } + if !spanning { + t.Error("a fabric was available and no multi-node fit was offered") + } +} + +func TestTensorWidthCannotExceedTheKVHeads(t *testing.T) { + // Grouped-query attention leaves far fewer KV heads than attention heads, + // and an engine refuses a width that cannot divide them. Offering tp=16 for + // a model with 8 KV heads is a plan that fails at launch. + for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) { + if f.TP > llama70B.KVHeads { + t.Errorf("tp=%d exceeds the model's %d KV heads", f.TP, llama70B.KVHeads) + } + } +} + +func TestOnlyPowersOfTwoAreOffered(t *testing.T) { + for _, f := range Fits(llama70B, []Hardware{h100Node(4, 6, "ib-a")}, 8192, []Precision{Q4GGUF}) { + if f.TP&(f.TP-1) != 0 { + t.Errorf("tp=%d is not a power of two", f.TP) + } + } +} + +func TestACardIsNeverPlannedToTheBrim(t *testing.T) { + // A plan that fills a card completely is one that loads on a good day. + // This matches vLLM's own 0.90 default deliberately: a planner that + // disagrees with the engine it plans for is worse than none. + for _, f := range Fits(coder14B, []Hardware{h100Node(1, 8, "")}, 32768, []Precision{BF16}) { + if f.PerGPUGB > 80*usableFraction+0.01 { + t.Errorf("planned %.1f GB on an 80 GB card: %s", f.PerGPUGB, f.Describe()) + } + if f.Headroom < 0 { + t.Errorf("negative headroom: %s", f.Describe()) + } + } +} + +func TestACPUNodeCanServeASmallModel(t *testing.T) { + // How this cluster began, and a reasonable way to try llmgw before + // committing cards to it. + cpu := Hardware{HostMemGB: 32} + fits := Fits(coder3B, []Hardware{cpu}, 32768, []Precision{Q4GGUF}) + if len(fits) == 0 { + t.Fatal("a 3B at q4 did not fit a 32 GB CPU node") + } + if !fits[0].CPUOnly { + t.Error("the fit is not marked CPU-only") + } + if !strings.Contains(fits[0].Describe(), "CPU") { + t.Errorf("the description does not say it is CPU: %q", fits[0].Describe()) + } + // And a 70B does not, whatever the quantisation. + if big := Fits(llama70B, []Hardware{cpu}, 8192, []Precision{Q4GGUF}); len(big) != 0 { + t.Errorf("a 70B was reported as fitting 32 GB of RAM: %s", big[0].Describe()) + } +} + +func TestNothingFitsIsAnEmptyAnswerNotAWrongOne(t *testing.T) { + tiny := Hardware{Class: "t4-16g", GPUMemGB: 16, GPUsPerNode: 1, Nodes: 1} + if fits := Fits(llama70B, []Hardware{tiny}, 32768, []Precision{BF16}); len(fits) != 0 { + t.Errorf("70B bf16 fitted a single 16 GB card: %s", fits[0].Describe()) + } +} diff --git a/internal/modelplan/huggingface.go b/internal/modelplan/huggingface.go new file mode 100644 index 00000000..9c6d381a --- /dev/null +++ b/internal/modelplan/huggingface.go @@ -0,0 +1,135 @@ +package modelplan + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// FetchModel reads a model's architecture from its repository. +// +// Read rather than asked for, because every field here is one somebody would +// otherwise have to look up and retype: the layer count, the KV head count, the +// head dimension and the parameter total. Those are exactly the numbers a +// person gets subtly wrong — attention heads instead of KV heads being the +// classic, and worth a factor of five on a model with grouped-query attention. +// +// Two documents: config.json for the architecture, and the repository index for +// the parameter count, which is the one number config.json does not carry. +func FetchModel(ctx context.Context, repo string) (Model, error) { + base := "https://huggingface.co" + cfg, err := fetchConfig(ctx, base, repo) + if err != nil { + return Model{}, err + } + m, err := cfg.toModel(repo) + if err != nil { + return Model{}, err + } + // Best effort. A repository that does not publish its parameter count is + // still usable — the caller can supply one — and failing the whole lookup + // over it would be worse than saying which number is missing. + if n, err := fetchParamCount(ctx, base, repo); err == nil && n > 0 { + m.Params = n + } + return m, nil +} + +type hfConfig struct { + NumHiddenLayers int `json:"num_hidden_layers"` + NumAttentionHeads int `json:"num_attention_heads"` + NumKeyValueHeads int `json:"num_key_value_heads"` + HiddenSize int `json:"hidden_size"` + HeadDim int `json:"head_dim"` + MaxPositions int `json:"max_position_embeddings"` + ModelType string `json:"model_type"` +} + +func (c hfConfig) toModel(repo string) (Model, error) { + if c.NumHiddenLayers == 0 { + return Model{}, fmt.Errorf("%s: config.json has no num_hidden_layers; is it a text model?", repo) + } + m := Model{ + Name: repo[strings.LastIndex(repo, "/")+1:], + Layers: c.NumHiddenLayers, + MaxContext: c.MaxPositions, + } + + // Grouped-query attention: when the KV head count is absent the model has + // as many as it has attention heads, which is multi-head attention and the + // older, more expensive arrangement. + m.KVHeads = c.NumKeyValueHeads + if m.KVHeads == 0 { + m.KVHeads = c.NumAttentionHeads + } + + // head_dim is explicit in newer configs and derived in older ones. Derived + // from the attention heads, never the KV heads: the hidden size is split + // across the query heads. + m.HeadDim = c.HeadDim + if m.HeadDim == 0 { + if c.NumAttentionHeads == 0 { + return Model{}, fmt.Errorf("%s: config.json gives neither head_dim nor num_attention_heads", repo) + } + m.HeadDim = c.HiddenSize / c.NumAttentionHeads + } + return m, nil +} + +func fetchConfig(ctx context.Context, base, repo string) (hfConfig, error) { + var c hfConfig + body, err := get(ctx, base+"/"+repo+"/resolve/main/config.json") + if err != nil { + return c, fmt.Errorf("reading %s: %w", repo, err) + } + if err := json.Unmarshal(body, &c); err != nil { + return c, fmt.Errorf("%s: config.json did not parse: %w", repo, err) + } + return c, nil +} + +// fetchParamCount reads the parameter total from the repository's own index. +// +// Taken from the safetensors index rather than estimated from the architecture, +// because an estimate is wrong by whatever the embeddings and the tied weights +// come to, and that error lands directly on the weights figure — the largest +// term in the whole calculation. +func fetchParamCount(ctx context.Context, base, repo string) (int64, error) { + body, err := get(ctx, base+"/api/models/"+repo) + if err != nil { + return 0, err + } + var info struct { + Safetensors struct { + Total int64 `json:"total"` + } `json:"safetensors"` + } + if err := json.Unmarshal(body, &info); err != nil { + return 0, err + } + return info.Safetensors.Total, nil +} + +func get(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + // A private or gated repository is the common case here, and "404" on + // its own sends people looking for a typo they did not make. + return nil, fmt.Errorf("%s returned %d; a private or gated repository needs to be downloaded by hand", + url, resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) +} diff --git a/internal/modelplan/huggingface_test.go b/internal/modelplan/huggingface_test.go new file mode 100644 index 00000000..94fc5c2b --- /dev/null +++ b/internal/modelplan/huggingface_test.go @@ -0,0 +1,77 @@ +package modelplan + +import "testing" + +func TestKVHeadsFallBackToAttentionHeads(t *testing.T) { + // A config without num_key_value_heads is multi-head attention, where every + // attention head keeps its own cache. Defaulting to zero instead would make + // the KV cache vanish from the calculation entirely. + m, err := hfConfig{ + NumHiddenLayers: 32, NumAttentionHeads: 32, HiddenSize: 4096, + MaxPositions: 4096, + }.toModel("org/old-model") + if err != nil { + t.Fatal(err) + } + if m.KVHeads != 32 { + t.Errorf("KVHeads = %d, want the attention head count", m.KVHeads) + } + if m.KVBytesPerToken(BF16) == 0 { + t.Error("the KV cache came out as nothing") + } +} + +func TestHeadDimIsDerivedFromAttentionHeadsNotKVHeads(t *testing.T) { + // The hidden size is split across the query heads. Dividing by the KV heads + // instead inflates head_dim by the grouping factor — eight, on a typical + // model — and the error lands straight on the cache figure. + m, err := hfConfig{ + NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8, + HiddenSize: 5120, MaxPositions: 32768, + }.toModel("Qwen/Qwen2.5-Coder-14B-Instruct") + if err != nil { + t.Fatal(err) + } + if m.HeadDim != 128 { + t.Errorf("HeadDim = %d, want 5120/40 = 128", m.HeadDim) + } + // Which reproduces the figure measured on this cluster. + if got := m.KVBytesPerToken(BF16); got != 192*1024 { + t.Errorf("KV/token = %d, want %d", got, 192*1024) + } +} + +func TestAnExplicitHeadDimWins(t *testing.T) { + // Newer configs state it, and some models do not have hidden_size divisible + // by the head count — deriving it there is simply wrong. + m, err := hfConfig{ + NumHiddenLayers: 32, NumAttentionHeads: 32, NumKeyValueHeads: 8, + HiddenSize: 4096, HeadDim: 256, MaxPositions: 8192, + }.toModel("org/model") + if err != nil { + t.Fatal(err) + } + if m.HeadDim != 256 { + t.Errorf("HeadDim = %d; the explicit value should win over 4096/32", m.HeadDim) + } +} + +func TestAConfigWithoutLayersIsRefusedClearly(t *testing.T) { + // Pointing this at an embedding or vision repository is an easy mistake, + // and a plan built from zeros would be confidently meaningless. + _, err := hfConfig{MaxPositions: 512}.toModel("org/not-a-text-model") + if err == nil { + t.Fatal("a config with no layers was accepted") + } + if !contains(err.Error(), "num_hidden_layers") { + t.Errorf("the error does not name what is missing: %v", err) + } +} + +func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) { + m, _ := hfConfig{NumHiddenLayers: 1, NumAttentionHeads: 1, HiddenSize: 64}. + toModel("Qwen/Qwen2.5-Coder-14B-Instruct") + if m.Name != "Qwen2.5-Coder-14B-Instruct" { + t.Errorf("Name = %q", m.Name) + } +} diff --git a/internal/modelplan/size.go b/internal/modelplan/size.go index 90f244ad..83f7323e 100644 --- a/internal/modelplan/size.go +++ b/internal/modelplan/size.go @@ -63,24 +63,33 @@ func PrecisionByName(s string) (Precision, error) { // Demand is what one replica needs, in bytes. type Demand struct { - Weights int64 - KVCache int64 + Weights int64 + KVCache int64 + // Overhead is what the runtime costs on a single GPU. Across a tensor + // parallel group it is recomputed from each rank's share — see PerGPU. Overhead int64 } // Total is everything one replica needs across all its GPUs. func (d Demand) Total() int64 { return d.Weights + d.KVCache + d.Overhead } -// PerGPU divides the demand over a replica's GPUs. +// PerGPU is what one GPU of a replica must hold. // -// Tensor parallelism splits both the weights and the KV cache, so both divide. -// The overhead does not: each process pays its own runtime, CUDA context and -// activation buffers, so it is charged per GPU rather than shared. +// Tensor parallelism splits the weights and the KV cache, so both divide. The +// overhead does not divide — each process pays its own CUDA context and +// allocator — but neither is it constant: activations scale with the shard the +// rank actually holds, so it is recomputed from the divided weights. +// +// Charging the whole model's overhead to every rank was the first version of +// this, and it was badly wrong in exactly the case that matters. Llama-3.3-70B +// at bf16 across two 80 GB cards came out at 85.9 GB a card and was reported as +// not fitting, which is a configuration people run every day. func (d Demand) PerGPU(gpus int) int64 { if gpus <= 0 { gpus = 1 } - return (d.Weights+d.KVCache)/int64(gpus) + d.Overhead + n := int64(gpus) + return (d.Weights+d.KVCache)/n + overheadBytes(d.Weights/n) } // KVBytesPerToken is what one token of context costs, for the whole model. @@ -108,17 +117,21 @@ func (m Model) Needs(p Precision, context int) Demand { } } -// overheadBytes is what the runtime costs beyond weights and cache. +// overheadBytes is what the runtime costs beyond weights and cache, per GPU. // -// A flat allowance plus a fraction of the weights: the flat part is the CUDA -// context, the framework and its allocator, which a small model pays as much as -// a large one; the fraction stands in for activations and fragmentation, which -// do scale. Deliberately generous. A plan that says a model fits and is wrong -// costs a twenty-minute load and an OOM; one that says it does not quite fit -// costs a smaller quantisation. +// A flat allowance for the CUDA context, the framework and its allocator — +// which a small model pays as much as a large one — plus a small fraction of +// the shard for activations, which do scale with it. +// +// The fraction is two percent rather than the ten it started at, because the +// ten was counted twice. usableFraction already holds back a tenth of every +// card for exactly this, so charging another tenth on top made a plan reject +// Llama-3.3-70B at bf16 across two 80 GB cards — a configuration people run +// every day. Reserving slack in two places, each unaware of the other, reads as +// caution and behaves as a bug. func overheadBytes(weights int64) int64 { const flat = 2 << 30 // 2 GiB - return flat + weights/10 + return flat + weights/50 } // GB renders bytes the way people discuss hardware. diff --git a/internal/modelplan/size_test.go b/internal/modelplan/size_test.go new file mode 100644 index 00000000..946d34ff --- /dev/null +++ b/internal/modelplan/size_test.go @@ -0,0 +1,160 @@ +package modelplan + +import ( + "math" + "testing" +) + +// Real models, with the figures measured on this cluster rather than derived +// from a formula — the point of testing arithmetic is that it agrees with the +// world, and a test built from the same formula would agree with itself. +var ( + coder3B = Model{ + Name: "Qwen2.5-Coder-3B", Params: 3_090_000_000, + Layers: 36, KVHeads: 2, HeadDim: 128, MaxContext: 32768, + } + coder14B = Model{ + Name: "Qwen2.5-Coder-14B", Params: 14_770_000_000, + Layers: 48, KVHeads: 8, HeadDim: 128, MaxContext: 32768, + } + llama70B = Model{ + Name: "Llama-3.3-70B", Params: 70_600_000_000, + Layers: 80, KVHeads: 8, HeadDim: 128, MaxContext: 131072, + } +) + +func closeTo(t *testing.T, what string, got, want, tolerance float64) { + t.Helper() + if math.Abs(got-want) > tolerance { + t.Errorf("%s = %.2f, want %.2f (±%.2f)", what, got, want, tolerance) + } +} + +func TestKVCachePerTokenMatchesTheArchitecture(t *testing.T) { + // 192 KiB a token for the 14B is the number that made sense of what this + // cluster measured: 8192 of context across four llama.cpp slots is 2048 + // each, and six gigabytes of cache at the model's full 32k. + if got := coder14B.KVBytesPerToken(BF16); got != 192*1024 { + t.Errorf("14B KV/token = %d bytes, want %d", got, 192*1024) + } + // Grouped-query attention is why this is KVHeads and not the attention + // head count. The 14B has 40 attention heads and 8 KV heads, and paying for + // 40 would overstate the cache fivefold. + if got := coder3B.KVBytesPerToken(BF16); got != 36*1024 { + t.Errorf("3B KV/token = %d bytes, want %d", got, 36*1024) + } + if got := llama70B.KVBytesPerToken(BF16); got != 320*1024 { + t.Errorf("70B KV/token = %d bytes, want %d", got, 320*1024) + } +} + +func TestQuantisedWeightsMatchTheFilesOnDisk(t *testing.T) { + // Calibrated against the GGUFs downloaded to this cluster, so the + // bytes-per-parameter figures are measurements rather than folklore. + // In GiB, which is what llama-bench itself reported for these files — + // "14.62 GiB" and "8.37 GiB". The download sizes quoted in decimal GB are + // the same bytes in different units, and mixing the two is how a plan ends + // up seven percent optimistic. + for _, tc := range []struct { + p Precision + want float64 // GiB, as llama.cpp accounts for it + }{ + {Q8GGUF, 14.62}, + {Q4GGUF, 8.37}, + } { + got := GB(coder14B.Needs(tc.p, 0).Weights) + closeTo(t, "14B "+tc.p.Name+" weights", got, tc.want, 0.35) + } + // And full precision, which is two bytes a parameter by definition. + closeTo(t, "14B bf16 weights", GB(coder14B.Needs(BF16, 0).Weights), 27.5, 0.6) +} + +func TestContextDominatesAtLongWindows(t *testing.T) { + // The reason KV cache is worth computing rather than hand-waving: at 128k + // of pooled context a 70B's cache is a substantial fraction of its weights, + // and a plan that ignores it fits a model that then will not load. + short := llama70B.Needs(BF16, 4096) + long := llama70B.Needs(BF16, 131072) + if short.Weights != long.Weights { + t.Fatal("weights should not depend on context") + } + if GB(long.KVCache) < 35 { + t.Errorf("70B KV at 128k = %.1f GB, expected tens of GB", GB(long.KVCache)) + } + if long.KVCache <= short.KVCache*8 { + t.Error("KV cache should scale with the context pool") + } +} + +func TestTensorParallelismDividesWeightsAndCacheButNotOverhead(t *testing.T) { + // Each rank holds its share of the weights and of the KV heads, so both + // divide. The runtime does not: every process pays its own CUDA context and + // activation buffers, which is why eight-way parallelism does not make a + // model eight times cheaper per card. + d := llama70B.Needs(BF16, 32768) + + one := d.PerGPU(1) + eight := d.PerGPU(8) + if eight >= one { + t.Fatal("splitting across GPUs should reduce the per-GPU demand") + } + // Strictly more than an eighth, because the overhead is not shared. + if eight <= one/8 { + t.Errorf("per-GPU at tp=8 is %d, at or below a clean eighth of %d: overhead is being divided", + eight, one) + } + if d.PerGPU(0) != d.PerGPU(1) { + t.Error("a nonsensical GPU count should be treated as one, not divided by zero") + } +} + +func TestA70BNeedsMoreThanOneCard(t *testing.T) { + // The question someone with hardware actually asks, and the answer that is + // easy to get wrong by hand: 70B at bf16 does not fit an 80 GB card, and + // does fit two. + d := llama70B.Needs(BF16, 32768) + const h100 = 80.0 + + if GB(d.PerGPU(1)) <= h100 { + t.Errorf("70B bf16 reported as fitting one 80 GB card (%.1f GB)", GB(d.PerGPU(1))) + } + if GB(d.PerGPU(2)) > h100 { + t.Errorf("70B bf16 across two cards is %.1f GB each, which should fit 80", GB(d.PerGPU(2))) + } +} + +func TestOverheadIsChargedEvenWithNoContext(t *testing.T) { + // A plan that forgets the runtime fits a model into exactly its weights and + // then watches it fail to load. + d := coder3B.Needs(BF16, 0) + if d.Overhead <= 0 { + t.Error("no overhead was charged") + } + if d.Total() <= d.Weights { + t.Error("the total is not more than the weights alone") + } +} + +func TestPrecisionByNameRefusesWhatItDoesNotKnow(t *testing.T) { + if _, err := PrecisionByName("bf16"); err != nil { + t.Errorf("bf16 was refused: %v", err) + } + _, err := PrecisionByName("fp3") + if err == nil { + t.Fatal("an unknown precision was accepted") + } + // The error lists what would work, because the caller mistyped something + // and a bare refusal makes them go looking. + if got := err.Error(); !contains(got, "q4_k_m") || !contains(got, "bf16") { + t.Errorf("the error does not say what is known: %q", got) + } +} + +func contains(s, sub string) bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +}