Work out whether a model fits the cluster, and how to split it

llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct

The arithmetic this replaces is the last thing between having GPUs and
serving a model on them. Choosing tp, gpu_class and a context that will
actually fit means knowing the layer count, the KV head count, what the
weights weigh at a chosen quantisation and how much of a card the runtime
keeps — and getting it wrong is not an error message, it is an
out-of-memory twenty minutes into a load. I computed those by hand for
every model on this cluster.

Both halves are read rather than retyped: the architecture from the model's
own config.json and parameter count, the hardware from the running fleet.
Attention heads in place of KV heads is the classic slip and is worth a
factor of five on a model with grouped-query attention, so it is not a
number anyone should be entering.

Every candidate is listed, not one answer. With cards to spare you want the
full-precision fit; with one card free you want to know a quantised one
exists. Choosing between those is not the planner's job.

The tests are calibrated against files actually downloaded here — llama.cpp
reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the
model reproduces both — and they caught two errors in my own arithmetic.
The overhead was charged per rank from the whole model's weights rather
than each rank's share, and it reserved a tenth of a card that
usableFraction had already reserved. Between them, Llama-3.3-70B at bf16
across two 80 GB cards came out as not fitting, which is a configuration
people run every day. Slack reserved in two places, each unaware of the
other, reads as caution and behaves as a bug.

Pools are grouped by class and fabric together, because a replica may never
span two fabrics: eight free cards split across two of them are not eight
cards for a model that needs eight, and merging them would plan something
that starts, passes its health checks and hangs on its first all-reduce.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 07:46:53 +02:00
1 parent ea4dd37f80
commit 300e298053
9 files changed
+1029 -16

No files matched your search

+1 -1
View File
@@ -25,7 +25,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client))
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
return c
}
+227
View File
@@ -0,0 +1,227 @@
package cli
import (
"fmt"
"strings"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// newModelPlanCmd answers whether a model fits this cluster, and how.
//
// The arithmetic it replaces is the last thing standing between having GPUs and
// serving a model on them. Deciding tp, gpu_class and a context that will
// actually fit means knowing the model's layer count, its KV head count, what
// its weights weigh at a chosen quantisation, and how much of a card the
// runtime keeps — and getting it wrong is not an error message, it is an
// out-of-memory twenty minutes into a load.
//
// It reads the model from its own repository and the hardware from the running
// fleet, so neither half is a number somebody retyped.
func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
var (
context int
params int64
precs []string
)
c := &cobra.Command{
Use: "plan <huggingface-repo>",
Short: "Work out whether a model fits this cluster, and how it should be split",
Long: "Reads the model's architecture from its repository and the hardware from\n" +
"the running fleet, then prints every way it could run — fewest GPUs\n" +
"first — and a catalog entry for the best one.\n\n" +
"Every candidate is listed rather than one answer, because the choice is\n" +
"not the planner's to make: with cards to spare you want the\n" +
"full-precision fit, and with one card free you want to know a quantised\n" +
"one exists.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
repo := args[0]
m, err := modelplan.FetchModel(cmd.Context(), repo)
if err != nil {
return err
}
if params > 0 {
m.Params = params
}
if m.Params == 0 {
return fmt.Errorf("%s does not publish a parameter count; pass --params", repo)
}
var chosen []modelplan.Precision
for _, name := range precs {
p, err := modelplan.PrecisionByName(name)
if err != nil {
return err
}
chosen = append(chosen, p)
}
var nodes []adminapi.NodeView
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
return err
}
hw := hardwareFrom(nodes)
if len(hw) == 0 {
return fmt.Errorf("no node is ready, so there is nothing to plan against")
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s: %.1fB parameters, %d layers, %d KV heads, trained to %d tokens\n",
m.Name, float64(m.Params)/1e9, m.Layers, m.KVHeads, m.MaxContext)
want := context
if want <= 0 {
want = m.MaxContext
}
fmt.Fprintf(out, "Planned for a %d-token pool. %s a token of context.\n\n",
want, humanBytes(m.KVBytesPerToken(modelplan.BF16)))
fits := modelplan.Fits(m, hw, want, chosen)
if len(fits) == 0 {
fmt.Fprintln(out, "It does not fit this cluster at any offered precision.")
fmt.Fprintln(out, "\nWhat would change that:")
fmt.Fprintln(out, " - a smaller context pool, with --context")
fmt.Fprintln(out, " - a heavier quantisation, with --precision q4_k_m")
fmt.Fprintln(out, " - more cards free; llmgw ctl status shows what is in use")
return errDoesNotFit
}
w := tab()
fmt.Fprintln(w, " GPUS\tPRECISION\tPER CARD\tFREE\tWHERE")
for _, f := range fits {
where := f.Class
if f.CPUOnly {
where = "cpu"
} else if f.Nodes > 1 {
where = fmt.Sprintf("%s, %d nodes on %s", f.Class, f.Nodes, f.Fabric)
}
fmt.Fprintf(w, " %d\t%s\t%.1f GB\t%.1f GB\t%s\n",
f.TP, f.Precision.Name, f.PerGPUGB, f.Headroom, where)
}
_ = w.Flush()
fmt.Fprintf(out, "\nA catalog entry for the first of those:\n\n%s\n",
catalogEntry(m, fits[0], repo, want))
fmt.Fprintln(out, "Save it and add it with: llmgw ctl model add -f <file>")
return nil
},
}
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
c.Flags().Int64Var(&params, "params", 0, "parameter count, when the repository does not publish one")
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
return c
}
// errDoesNotFit gives a script something to test without parsing prose.
var errDoesNotFit = fmt.Errorf("the model does not fit this cluster")
// hardwareFrom turns the fleet into pools a replica could actually occupy.
//
// Grouped by class and fabric together, because a replica may never span two
// fabrics: eight free cards split across two fabrics are not eight cards for a
// model that needs eight, and treating them as one pool would plan something
// that hangs on its first all-reduce.
func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
type key struct{ class, fabric string }
pools := map[key]*modelplan.Hardware{}
for _, n := range nodes {
if n.State != "ready" {
continue
}
fabric := ""
if len(n.Fabrics) > 0 {
fabric = n.Fabrics[0]
}
class := ""
if len(n.Classes) > 0 {
class = n.Classes[0]
}
k := key{class, fabric}
if pools[k] == nil {
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
}
p := pools[k]
p.Nodes++
// The widest a single node could contribute, not the total: a tensor
// group takes the same number of cards from every node it spans.
if n.GPUFree > p.GPUsPerNode {
p.GPUsPerNode = n.GPUFree
}
if n.MemoryGB > p.HostMemGB {
p.HostMemGB = n.MemoryGB
}
// Card memory from the class label, which carries it by construction:
// "h100-80g" is eighty gigabytes and says so.
if class != "" && p.GPUMemGB == 0 {
p.GPUMemGB = memFromClass(class)
}
}
out := make([]modelplan.Hardware, 0, len(pools))
for _, p := range pools {
if p.Class != "" && (p.GPUMemGB == 0 || p.GPUsPerNode == 0) {
continue // a GPU pool with no free cards is not a pool
}
out = append(out, *p)
}
return out
}
// memFromClass reads the card size out of a class label such as "h100-80g".
func memFromClass(class string) int {
i := strings.LastIndex(class, "-")
if i < 0 || !strings.HasSuffix(class, "g") {
return 0
}
var gb int
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
return 0
}
return gb
}
// catalogEntry renders a fit as the YAML that would serve it.
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
var b strings.Builder
name := strings.ToLower(m.Name)
fmt.Fprintf(&b, "name: %s\n", name)
fmt.Fprintf(&b, "source: %s\n", source)
fmt.Fprintf(&b, "routing_profile: chat\n")
fmt.Fprintf(&b, "context_length: %d\n", context)
fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n")
fmt.Fprintf(&b, "profiles:\n")
switch {
case f.CPUOnly:
fmt.Fprintf(&b, " - name: cpu\n engine: llamacpp\n")
fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2)
default:
fmt.Fprintf(&b, " - name: tp%d\n engine: vllm\n", f.TP)
fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP)
if f.Nodes > 1 {
fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode)
}
}
fmt.Fprintf(&b, " priority: 100\n weight: 1\n")
// Measured rather than guessed is the rule everywhere else in this system,
// and a planner should say so rather than pretend its estimate is one.
fmt.Fprintf(&b, " startup_timeout: 10m\n")
b.WriteString("\n# weight should come from `llmgw ctl bench` once it is running;\n")
b.WriteString("# 1 is a placeholder, not a measurement.\n")
return b.String()
}
// humanBytes renders a per-token cache size, which is always small.
func humanBytes(n int64) string {
switch {
case n >= 1<<20:
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
default:
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
}
}
+84
View File
@@ -59,3 +59,87 @@ func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) {
})
}
}
func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) {
// A replica may never span two fabrics, so eight free cards split across
// two of them are not eight cards for a model that needs eight. Merging
// them would plan something that starts, passes health checks and hangs on
// its first all-reduce.
nodes := []adminapi.NodeView{
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
{ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
{ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4},
}
hw := hardwareFrom(nodes)
if len(hw) != 2 {
t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw)
}
for _, p := range hw {
switch p.Fabric {
case "ib-a":
if p.Nodes != 2 {
t.Errorf("ib-a has %d nodes, want 2", p.Nodes)
}
case "ib-b":
if p.Nodes != 1 {
t.Errorf("ib-b has %d nodes, want 1", p.Nodes)
}
default:
t.Errorf("unexpected fabric %q", p.Fabric)
}
}
}
func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) {
// Planning against a node that is draining or unreachable produces a fit
// that cannot be placed, which is worse than saying nothing fits.
nodes := []adminapi.NodeView{
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
{ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
{ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
}
hw := hardwareFrom(nodes)
if len(hw) != 1 || hw[0].Nodes != 1 {
t.Fatalf("expected only the ready node, got %+v", hw)
}
}
func TestCardMemoryComesFromTheClassLabel(t *testing.T) {
// The label carries it by construction — "h100-80g" is eighty gigabytes and
// says so — which is why the planner needs no table of card models.
for label, want := range map[string]int{
"h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24,
"nonsense": 0, "": 0, "h100-": 0,
} {
if got := memFromClass(label); got != want {
t.Errorf("memFromClass(%q) = %d, want %d", label, got, want)
}
}
}
func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) {
// How this cluster began: a node with no cards is not an empty pool, it is
// a pool measured in RAM.
hw := hardwareFrom([]adminapi.NodeView{
{ID: "cpu1", State: "ready", MemoryGB: 31},
})
if len(hw) != 1 {
t.Fatalf("the CPU node produced %d pools", len(hw))
}
if hw[0].Class != "" || hw[0].HostMemGB != 31 {
t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0])
}
}
func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
// Fully allocated hardware is not capacity, and offering it would plan a
// replica that cannot be placed.
hw := hardwareFrom([]adminapi.NodeView{
{ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8},
})
for _, p := range hw {
if p.Class != "" {
t.Errorf("a node with no free cards became a pool: %+v", p)
}
}
}
+167
View File
@@ -0,0 +1,167 @@
package modelplan
import (
"fmt"
"sort"
)
// Hardware is what one kind of node offers, as the fleet reports it.
type Hardware struct {
// Class is the GPU class, e.g. "h100-80g". Empty means a CPU-only node.
Class string
// GPUMemGB is one card's memory.
GPUMemGB int
// GPUsPerNode is how many cards a node of this kind has free.
GPUsPerNode int
// Nodes is how many such nodes are available.
Nodes int
// Fabric is what they are attached to, empty when they are not. A replica
// may never span two fabrics, so nodes on different ones are separate
// pools however identical their cards.
Fabric string
// HostMemGB is the machine's RAM, which is what a CPU profile needs.
HostMemGB int
}
// Fit is one way a model could run.
type Fit struct {
Precision Precision
Class string
TP int
Nodes int
GPUsPerNode int
Fabric string
// Context is the token pool this fit was checked against.
Context int
// PerGPUGB is what each card must hold, and Headroom what is left of it.
PerGPUGB float64
Headroom float64
// CPUOnly marks a fit that needs no GPU at all.
CPUOnly bool
}
// usableFraction is how much of a card a plan may assume it can have.
//
// Not all of it. A framework's allocator fragments, the driver reserves some,
// and a card planned to ninety-nine percent full is a card that fails to load
// on a bad day rather than a good one. vLLM's own default GPU utilisation is
// 0.90 and this matches it deliberately: a plan that disagrees with the engine
// it is planning for is worse than no plan.
const usableFraction = 0.90
// Fits returns every way this model could run on the given hardware, best
// first — fewest GPUs, then fewest nodes, then highest precision.
//
// Every candidate, not one answer, because the choice between them is not the
// planner's to make: a team with spare cards wants the full-precision fit, and
// a team with one card free wants to know a quantised one exists. Returning
// only the "best" would hide the option someone actually wants.
func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit {
if context <= 0 {
context = m.MaxContext
}
if len(precisions) == 0 {
precisions = []Precision{BF16, FP8, Q8GGUF, Q4GGUF}
}
var out []Fit
for _, p := range precisions {
d := m.Needs(p, context)
for _, h := range hw {
out = append(out, fitsOn(m, h, d, p, context)...)
}
}
sort.SliceStable(out, func(i, j int) bool {
a, b := out[i], out[j]
if ga, gb := a.TP, b.TP; ga != gb {
return ga < gb
}
if a.Nodes != b.Nodes {
return a.Nodes < b.Nodes
}
// Higher precision first among equals: the same hardware serving a
// better model is strictly the better answer.
return a.Precision.BytesPerParam > b.Precision.BytesPerParam
})
return out
}
func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
if h.Class == "" {
return cpuFit(h, d, p, context)
}
var out []Fit
budget := float64(h.GPUMemGB) * usableFraction
// Powers of two only. Tensor parallelism divides attention heads across
// ranks, and an engine that cannot divide them evenly refuses the profile —
// so offering tp=3 would produce a plan that fails at launch.
for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 {
nodes := 1
perNode := width
if width > h.GPUsPerNode {
perNode = h.GPUsPerNode
nodes = width / h.GPUsPerNode
}
if nodes > h.Nodes {
break
}
// A replica spanning nodes needs a fabric. Without one every node is
// its own island, and a multi-node replica would hang on its first
// all-reduce rather than fail to start.
if nodes > 1 && h.Fabric == "" {
continue
}
// The KV heads have to divide too, and they are far fewer than the
// attention heads under grouped-query attention: a model with 8 KV
// heads cannot be split sixteen ways.
if m.KVHeads > 0 && width > m.KVHeads {
break
}
per := GB(d.PerGPU(width))
if per > budget {
continue
}
out = append(out, Fit{
Precision: p, Class: h.Class, TP: width, Nodes: nodes,
GPUsPerNode: perNode, Fabric: h.Fabric, Context: context,
PerGPUGB: per, Headroom: budget - per,
})
}
return out
}
// cpuFit checks a node with no cards, where the whole working set lives in RAM.
func cpuFit(h Hardware, d Demand, p Precision, context int) []Fit {
if h.HostMemGB <= 0 {
return nil
}
// Less of the machine than of a card: the operating system, the page cache
// and everything else on the node want their share, and a CPU node is
// rarely dedicated the way a GPU node is.
budget := float64(h.HostMemGB) * 0.75
need := GB(d.Total())
if need > budget {
return nil
}
return []Fit{{
Precision: p, TP: 1, Nodes: 1, Context: context,
PerGPUGB: need, Headroom: budget - need, CPUOnly: true,
}}
}
// Describe renders a fit the way someone would say it out loud.
func (f Fit) Describe() string {
if f.CPUOnly {
return fmt.Sprintf("CPU, %s, %.1f GB of RAM", f.Precision.Name, f.PerGPUGB)
}
where := fmt.Sprintf("%d×%s", f.TP, f.Class)
if f.Nodes > 1 {
where = fmt.Sprintf("%d nodes × %d×%s on %s", f.Nodes, f.GPUsPerNode, f.Class, f.Fabric)
}
return fmt.Sprintf("%s, %s, %.1f GB a card (%.1f free)",
where, f.Precision.Name, f.PerGPUGB, f.Headroom)
}
+150
View File
@@ -0,0 +1,150 @@
package modelplan
import (
"strings"
"testing"
)
func h100Node(nodes, gpus int, fabric string) Hardware {
return Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: gpus, Nodes: nodes, Fabric: fabric}
}
func TestASmallModelFitsOneCard(t *testing.T) {
fits := Fits(coder3B, []Hardware{h100Node(4, 8, "ib-a")}, 32768, []Precision{BF16})
if len(fits) == 0 {
t.Fatal("a 3B was reported as not fitting an H100")
}
// Best first, and the best answer for a small model is one card.
if fits[0].TP != 1 {
t.Errorf("best fit uses %d GPUs; a 3B needs one", fits[0].TP)
}
}
func TestA70BNeedsTwoCardsAtFullPrecision(t *testing.T) {
// The configuration people run every day, and the one an earlier overhead
// model wrongly called impossible by reserving slack twice.
fits := Fits(llama70B, []Hardware{h100Node(4, 8, "ib-a")}, 8192, []Precision{BF16})
if len(fits) == 0 {
t.Fatal("70B bf16 reported as not fitting eight H100s")
}
if fits[0].TP != 2 {
t.Errorf("best fit is tp=%d; 70B at bf16 on two 80 GB cards is routine", fits[0].TP)
}
for _, f := range fits {
if f.TP == 1 {
t.Errorf("70B bf16 reported as fitting a single card: %s", f.Describe())
}
}
}
func TestContextChangesHowManyCardsAModelNeeds(t *testing.T) {
// The fact that makes this worth computing rather than looking up. The same
// 70B at the same precision needs two cards for an 8k pool and four for
// 32k, because the KV cache is what grows — and "70B needs two H100s" is
// repeated everywhere without the context it was true for.
hw := []Hardware{h100Node(4, 8, "ib-a")}
short := Fits(llama70B, hw, 8192, []Precision{BF16})
long := Fits(llama70B, hw, 32768, []Precision{BF16})
if len(short) == 0 || len(long) == 0 {
t.Fatal("expected fits at both context sizes")
}
if long[0].TP <= short[0].TP {
t.Errorf("tp at 32k is %d and at 8k is %d; a larger context should need at least as much",
long[0].TP, short[0].TP)
}
}
func TestQuantisationOpensUpSmallerHardware(t *testing.T) {
// The answer someone with one spare card needs: not "no", but "not at this
// precision".
one := []Hardware{h100Node(1, 1, "")}
if full := Fits(llama70B, one, 8192, []Precision{BF16}); len(full) != 0 {
t.Errorf("70B bf16 fitted a single card: %s", full[0].Describe())
}
if quant := Fits(llama70B, one, 8192, []Precision{Q4GGUF}); len(quant) == 0 {
t.Error("70B at q4 did not fit a single 80 GB card, which it should")
}
}
func TestATensorGroupNeverSpansFabrics(t *testing.T) {
// Two nodes with no fabric are two islands. A multi-node replica placed
// across them would start, pass health checks and hang on its first
// all-reduce, which is worse than refusing to plan it.
noFabric := Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: 2, Nodes: 4}
for _, f := range Fits(llama70B, []Hardware{noFabric}, 8192, []Precision{BF16}) {
if f.Nodes > 1 {
t.Errorf("planned %d nodes with no fabric: %s", f.Nodes, f.Describe())
}
}
withFabric := noFabric
withFabric.Fabric = "ib-a"
var spanning bool
for _, f := range Fits(llama70B, []Hardware{withFabric}, 8192, []Precision{BF16}) {
if f.Nodes > 1 {
spanning = true
}
}
if !spanning {
t.Error("a fabric was available and no multi-node fit was offered")
}
}
func TestTensorWidthCannotExceedTheKVHeads(t *testing.T) {
// Grouped-query attention leaves far fewer KV heads than attention heads,
// and an engine refuses a width that cannot divide them. Offering tp=16 for
// a model with 8 KV heads is a plan that fails at launch.
for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) {
if f.TP > llama70B.KVHeads {
t.Errorf("tp=%d exceeds the model's %d KV heads", f.TP, llama70B.KVHeads)
}
}
}
func TestOnlyPowersOfTwoAreOffered(t *testing.T) {
for _, f := range Fits(llama70B, []Hardware{h100Node(4, 6, "ib-a")}, 8192, []Precision{Q4GGUF}) {
if f.TP&(f.TP-1) != 0 {
t.Errorf("tp=%d is not a power of two", f.TP)
}
}
}
func TestACardIsNeverPlannedToTheBrim(t *testing.T) {
// A plan that fills a card completely is one that loads on a good day.
// This matches vLLM's own 0.90 default deliberately: a planner that
// disagrees with the engine it plans for is worse than none.
for _, f := range Fits(coder14B, []Hardware{h100Node(1, 8, "")}, 32768, []Precision{BF16}) {
if f.PerGPUGB > 80*usableFraction+0.01 {
t.Errorf("planned %.1f GB on an 80 GB card: %s", f.PerGPUGB, f.Describe())
}
if f.Headroom < 0 {
t.Errorf("negative headroom: %s", f.Describe())
}
}
}
func TestACPUNodeCanServeASmallModel(t *testing.T) {
// How this cluster began, and a reasonable way to try llmgw before
// committing cards to it.
cpu := Hardware{HostMemGB: 32}
fits := Fits(coder3B, []Hardware{cpu}, 32768, []Precision{Q4GGUF})
if len(fits) == 0 {
t.Fatal("a 3B at q4 did not fit a 32 GB CPU node")
}
if !fits[0].CPUOnly {
t.Error("the fit is not marked CPU-only")
}
if !strings.Contains(fits[0].Describe(), "CPU") {
t.Errorf("the description does not say it is CPU: %q", fits[0].Describe())
}
// And a 70B does not, whatever the quantisation.
if big := Fits(llama70B, []Hardware{cpu}, 8192, []Precision{Q4GGUF}); len(big) != 0 {
t.Errorf("a 70B was reported as fitting 32 GB of RAM: %s", big[0].Describe())
}
}
func TestNothingFitsIsAnEmptyAnswerNotAWrongOne(t *testing.T) {
tiny := Hardware{Class: "t4-16g", GPUMemGB: 16, GPUsPerNode: 1, Nodes: 1}
if fits := Fits(llama70B, []Hardware{tiny}, 32768, []Precision{BF16}); len(fits) != 0 {
t.Errorf("70B bf16 fitted a single 16 GB card: %s", fits[0].Describe())
}
}
+135
View File
@@ -0,0 +1,135 @@
package modelplan
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// FetchModel reads a model's architecture from its repository.
//
// Read rather than asked for, because every field here is one somebody would
// otherwise have to look up and retype: the layer count, the KV head count, the
// head dimension and the parameter total. Those are exactly the numbers a
// person gets subtly wrong — attention heads instead of KV heads being the
// classic, and worth a factor of five on a model with grouped-query attention.
//
// Two documents: config.json for the architecture, and the repository index for
// the parameter count, which is the one number config.json does not carry.
func FetchModel(ctx context.Context, repo string) (Model, error) {
base := "https://huggingface.co"
cfg, err := fetchConfig(ctx, base, repo)
if err != nil {
return Model{}, err
}
m, err := cfg.toModel(repo)
if err != nil {
return Model{}, err
}
// Best effort. A repository that does not publish its parameter count is
// still usable — the caller can supply one — and failing the whole lookup
// over it would be worse than saying which number is missing.
if n, err := fetchParamCount(ctx, base, repo); err == nil && n > 0 {
m.Params = n
}
return m, nil
}
type hfConfig struct {
NumHiddenLayers int `json:"num_hidden_layers"`
NumAttentionHeads int `json:"num_attention_heads"`
NumKeyValueHeads int `json:"num_key_value_heads"`
HiddenSize int `json:"hidden_size"`
HeadDim int `json:"head_dim"`
MaxPositions int `json:"max_position_embeddings"`
ModelType string `json:"model_type"`
}
func (c hfConfig) toModel(repo string) (Model, error) {
if c.NumHiddenLayers == 0 {
return Model{}, fmt.Errorf("%s: config.json has no num_hidden_layers; is it a text model?", repo)
}
m := Model{
Name: repo[strings.LastIndex(repo, "/")+1:],
Layers: c.NumHiddenLayers,
MaxContext: c.MaxPositions,
}
// Grouped-query attention: when the KV head count is absent the model has
// as many as it has attention heads, which is multi-head attention and the
// older, more expensive arrangement.
m.KVHeads = c.NumKeyValueHeads
if m.KVHeads == 0 {
m.KVHeads = c.NumAttentionHeads
}
// head_dim is explicit in newer configs and derived in older ones. Derived
// from the attention heads, never the KV heads: the hidden size is split
// across the query heads.
m.HeadDim = c.HeadDim
if m.HeadDim == 0 {
if c.NumAttentionHeads == 0 {
return Model{}, fmt.Errorf("%s: config.json gives neither head_dim nor num_attention_heads", repo)
}
m.HeadDim = c.HiddenSize / c.NumAttentionHeads
}
return m, nil
}
func fetchConfig(ctx context.Context, base, repo string) (hfConfig, error) {
var c hfConfig
body, err := get(ctx, base+"/"+repo+"/resolve/main/config.json")
if err != nil {
return c, fmt.Errorf("reading %s: %w", repo, err)
}
if err := json.Unmarshal(body, &c); err != nil {
return c, fmt.Errorf("%s: config.json did not parse: %w", repo, err)
}
return c, nil
}
// fetchParamCount reads the parameter total from the repository's own index.
//
// Taken from the safetensors index rather than estimated from the architecture,
// because an estimate is wrong by whatever the embeddings and the tied weights
// come to, and that error lands directly on the weights figure — the largest
// term in the whole calculation.
func fetchParamCount(ctx context.Context, base, repo string) (int64, error) {
body, err := get(ctx, base+"/api/models/"+repo)
if err != nil {
return 0, err
}
var info struct {
Safetensors struct {
Total int64 `json:"total"`
} `json:"safetensors"`
}
if err := json.Unmarshal(body, &info); err != nil {
return 0, err
}
return info.Safetensors.Total, nil
}
func get(ctx context.Context, url string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
// A private or gated repository is the common case here, and "404" on
// its own sends people looking for a typo they did not make.
return nil, fmt.Errorf("%s returned %d; a private or gated repository needs to be downloaded by hand",
url, resp.StatusCode)
}
return io.ReadAll(io.LimitReader(resp.Body, 4<<20))
}
+77
View File
@@ -0,0 +1,77 @@
package modelplan
import "testing"
func TestKVHeadsFallBackToAttentionHeads(t *testing.T) {
// A config without num_key_value_heads is multi-head attention, where every
// attention head keeps its own cache. Defaulting to zero instead would make
// the KV cache vanish from the calculation entirely.
m, err := hfConfig{
NumHiddenLayers: 32, NumAttentionHeads: 32, HiddenSize: 4096,
MaxPositions: 4096,
}.toModel("org/old-model")
if err != nil {
t.Fatal(err)
}
if m.KVHeads != 32 {
t.Errorf("KVHeads = %d, want the attention head count", m.KVHeads)
}
if m.KVBytesPerToken(BF16) == 0 {
t.Error("the KV cache came out as nothing")
}
}
func TestHeadDimIsDerivedFromAttentionHeadsNotKVHeads(t *testing.T) {
// The hidden size is split across the query heads. Dividing by the KV heads
// instead inflates head_dim by the grouping factor — eight, on a typical
// model — and the error lands straight on the cache figure.
m, err := hfConfig{
NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8,
HiddenSize: 5120, MaxPositions: 32768,
}.toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
if err != nil {
t.Fatal(err)
}
if m.HeadDim != 128 {
t.Errorf("HeadDim = %d, want 5120/40 = 128", m.HeadDim)
}
// Which reproduces the figure measured on this cluster.
if got := m.KVBytesPerToken(BF16); got != 192*1024 {
t.Errorf("KV/token = %d, want %d", got, 192*1024)
}
}
func TestAnExplicitHeadDimWins(t *testing.T) {
// Newer configs state it, and some models do not have hidden_size divisible
// by the head count — deriving it there is simply wrong.
m, err := hfConfig{
NumHiddenLayers: 32, NumAttentionHeads: 32, NumKeyValueHeads: 8,
HiddenSize: 4096, HeadDim: 256, MaxPositions: 8192,
}.toModel("org/model")
if err != nil {
t.Fatal(err)
}
if m.HeadDim != 256 {
t.Errorf("HeadDim = %d; the explicit value should win over 4096/32", m.HeadDim)
}
}
func TestAConfigWithoutLayersIsRefusedClearly(t *testing.T) {
// Pointing this at an embedding or vision repository is an easy mistake,
// and a plan built from zeros would be confidently meaningless.
_, err := hfConfig{MaxPositions: 512}.toModel("org/not-a-text-model")
if err == nil {
t.Fatal("a config with no layers was accepted")
}
if !contains(err.Error(), "num_hidden_layers") {
t.Errorf("the error does not name what is missing: %v", err)
}
}
func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) {
m, _ := hfConfig{NumHiddenLayers: 1, NumAttentionHeads: 1, HiddenSize: 64}.
toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
if m.Name != "Qwen2.5-Coder-14B-Instruct" {
t.Errorf("Name = %q", m.Name)
}
}
+28 -15
View File
@@ -63,24 +63,33 @@ func PrecisionByName(s string) (Precision, error) {
// Demand is what one replica needs, in bytes.
type Demand struct {
Weights int64
KVCache int64
Weights int64
KVCache int64
// Overhead is what the runtime costs on a single GPU. Across a tensor
// parallel group it is recomputed from each rank's share — see PerGPU.
Overhead int64
}
// Total is everything one replica needs across all its GPUs.
func (d Demand) Total() int64 { return d.Weights + d.KVCache + d.Overhead }
// PerGPU divides the demand over a replica's GPUs.
// PerGPU is what one GPU of a replica must hold.
//
// Tensor parallelism splits both the weights and the KV cache, so both divide.
// The overhead does not: each process pays its own runtime, CUDA context and
// activation buffers, so it is charged per GPU rather than shared.
// Tensor parallelism splits the weights and the KV cache, so both divide. The
// overhead does not divide — each process pays its own CUDA context and
// allocator — but neither is it constant: activations scale with the shard the
// rank actually holds, so it is recomputed from the divided weights.
//
// Charging the whole model's overhead to every rank was the first version of
// this, and it was badly wrong in exactly the case that matters. Llama-3.3-70B
// at bf16 across two 80 GB cards came out at 85.9 GB a card and was reported as
// not fitting, which is a configuration people run every day.
func (d Demand) PerGPU(gpus int) int64 {
if gpus <= 0 {
gpus = 1
}
return (d.Weights+d.KVCache)/int64(gpus) + d.Overhead
n := int64(gpus)
return (d.Weights+d.KVCache)/n + overheadBytes(d.Weights/n)
}
// KVBytesPerToken is what one token of context costs, for the whole model.
@@ -108,17 +117,21 @@ func (m Model) Needs(p Precision, context int) Demand {
}
}
// overheadBytes is what the runtime costs beyond weights and cache.
// overheadBytes is what the runtime costs beyond weights and cache, per GPU.
//
// A flat allowance plus a fraction of the weights: the flat part is the CUDA
// context, the framework and its allocator, which a small model pays as much as
// a large one; the fraction stands in for activations and fragmentation, which
// do scale. Deliberately generous. A plan that says a model fits and is wrong
// costs a twenty-minute load and an OOM; one that says it does not quite fit
// costs a smaller quantisation.
// A flat allowance for the CUDA context, the framework and its allocator —
// which a small model pays as much as a large one — plus a small fraction of
// the shard for activations, which do scale with it.
//
// The fraction is two percent rather than the ten it started at, because the
// ten was counted twice. usableFraction already holds back a tenth of every
// card for exactly this, so charging another tenth on top made a plan reject
// Llama-3.3-70B at bf16 across two 80 GB cards — a configuration people run
// every day. Reserving slack in two places, each unaware of the other, reads as
// caution and behaves as a bug.
func overheadBytes(weights int64) int64 {
const flat = 2 << 30 // 2 GiB
return flat + weights/10
return flat + weights/50
}
// GB renders bytes the way people discuss hardware.
+160
View File
@@ -0,0 +1,160 @@
package modelplan
import (
"math"
"testing"
)
// Real models, with the figures measured on this cluster rather than derived
// from a formula — the point of testing arithmetic is that it agrees with the
// world, and a test built from the same formula would agree with itself.
var (
coder3B = Model{
Name: "Qwen2.5-Coder-3B", Params: 3_090_000_000,
Layers: 36, KVHeads: 2, HeadDim: 128, MaxContext: 32768,
}
coder14B = Model{
Name: "Qwen2.5-Coder-14B", Params: 14_770_000_000,
Layers: 48, KVHeads: 8, HeadDim: 128, MaxContext: 32768,
}
llama70B = Model{
Name: "Llama-3.3-70B", Params: 70_600_000_000,
Layers: 80, KVHeads: 8, HeadDim: 128, MaxContext: 131072,
}
)
func closeTo(t *testing.T, what string, got, want, tolerance float64) {
t.Helper()
if math.Abs(got-want) > tolerance {
t.Errorf("%s = %.2f, want %.2f (±%.2f)", what, got, want, tolerance)
}
}
func TestKVCachePerTokenMatchesTheArchitecture(t *testing.T) {
// 192 KiB a token for the 14B is the number that made sense of what this
// cluster measured: 8192 of context across four llama.cpp slots is 2048
// each, and six gigabytes of cache at the model's full 32k.
if got := coder14B.KVBytesPerToken(BF16); got != 192*1024 {
t.Errorf("14B KV/token = %d bytes, want %d", got, 192*1024)
}
// Grouped-query attention is why this is KVHeads and not the attention
// head count. The 14B has 40 attention heads and 8 KV heads, and paying for
// 40 would overstate the cache fivefold.
if got := coder3B.KVBytesPerToken(BF16); got != 36*1024 {
t.Errorf("3B KV/token = %d bytes, want %d", got, 36*1024)
}
if got := llama70B.KVBytesPerToken(BF16); got != 320*1024 {
t.Errorf("70B KV/token = %d bytes, want %d", got, 320*1024)
}
}
func TestQuantisedWeightsMatchTheFilesOnDisk(t *testing.T) {
// Calibrated against the GGUFs downloaded to this cluster, so the
// bytes-per-parameter figures are measurements rather than folklore.
// In GiB, which is what llama-bench itself reported for these files —
// "14.62 GiB" and "8.37 GiB". The download sizes quoted in decimal GB are
// the same bytes in different units, and mixing the two is how a plan ends
// up seven percent optimistic.
for _, tc := range []struct {
p Precision
want float64 // GiB, as llama.cpp accounts for it
}{
{Q8GGUF, 14.62},
{Q4GGUF, 8.37},
} {
got := GB(coder14B.Needs(tc.p, 0).Weights)
closeTo(t, "14B "+tc.p.Name+" weights", got, tc.want, 0.35)
}
// And full precision, which is two bytes a parameter by definition.
closeTo(t, "14B bf16 weights", GB(coder14B.Needs(BF16, 0).Weights), 27.5, 0.6)
}
func TestContextDominatesAtLongWindows(t *testing.T) {
// The reason KV cache is worth computing rather than hand-waving: at 128k
// of pooled context a 70B's cache is a substantial fraction of its weights,
// and a plan that ignores it fits a model that then will not load.
short := llama70B.Needs(BF16, 4096)
long := llama70B.Needs(BF16, 131072)
if short.Weights != long.Weights {
t.Fatal("weights should not depend on context")
}
if GB(long.KVCache) < 35 {
t.Errorf("70B KV at 128k = %.1f GB, expected tens of GB", GB(long.KVCache))
}
if long.KVCache <= short.KVCache*8 {
t.Error("KV cache should scale with the context pool")
}
}
func TestTensorParallelismDividesWeightsAndCacheButNotOverhead(t *testing.T) {
// Each rank holds its share of the weights and of the KV heads, so both
// divide. The runtime does not: every process pays its own CUDA context and
// activation buffers, which is why eight-way parallelism does not make a
// model eight times cheaper per card.
d := llama70B.Needs(BF16, 32768)
one := d.PerGPU(1)
eight := d.PerGPU(8)
if eight >= one {
t.Fatal("splitting across GPUs should reduce the per-GPU demand")
}
// Strictly more than an eighth, because the overhead is not shared.
if eight <= one/8 {
t.Errorf("per-GPU at tp=8 is %d, at or below a clean eighth of %d: overhead is being divided",
eight, one)
}
if d.PerGPU(0) != d.PerGPU(1) {
t.Error("a nonsensical GPU count should be treated as one, not divided by zero")
}
}
func TestA70BNeedsMoreThanOneCard(t *testing.T) {
// The question someone with hardware actually asks, and the answer that is
// easy to get wrong by hand: 70B at bf16 does not fit an 80 GB card, and
// does fit two.
d := llama70B.Needs(BF16, 32768)
const h100 = 80.0
if GB(d.PerGPU(1)) <= h100 {
t.Errorf("70B bf16 reported as fitting one 80 GB card (%.1f GB)", GB(d.PerGPU(1)))
}
if GB(d.PerGPU(2)) > h100 {
t.Errorf("70B bf16 across two cards is %.1f GB each, which should fit 80", GB(d.PerGPU(2)))
}
}
func TestOverheadIsChargedEvenWithNoContext(t *testing.T) {
// A plan that forgets the runtime fits a model into exactly its weights and
// then watches it fail to load.
d := coder3B.Needs(BF16, 0)
if d.Overhead <= 0 {
t.Error("no overhead was charged")
}
if d.Total() <= d.Weights {
t.Error("the total is not more than the weights alone")
}
}
func TestPrecisionByNameRefusesWhatItDoesNotKnow(t *testing.T) {
if _, err := PrecisionByName("bf16"); err != nil {
t.Errorf("bf16 was refused: %v", err)
}
_, err := PrecisionByName("fp3")
if err == nil {
t.Fatal("an unknown precision was accepted")
}
// The error lists what would work, because the caller mistyped something
// and a bare refusal makes them go looking.
if got := err.Error(); !contains(got, "q4_k_m") || !contains(got, "bf16") {
t.Errorf("the error does not say what is known: %q", got)
}
}
func contains(s, sub string) bool {
for i := 0; i+len(sub) <= len(s); i++ {
if s[i:i+len(sub)] == sub {
return true
}
}
return false
}