Files
llmgw/internal/modelplan/fit.go
T
Ilker ManapandClaude Opus 5 300e298053
CI / test (push) Has been cancelled
Work out whether a model fits the cluster, and how to split it
llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct

The arithmetic this replaces is the last thing between having GPUs and
serving a model on them. Choosing tp, gpu_class and a context that will
actually fit means knowing the layer count, the KV head count, what the
weights weigh at a chosen quantisation and how much of a card the runtime
keeps — and getting it wrong is not an error message, it is an
out-of-memory twenty minutes into a load. I computed those by hand for
every model on this cluster.

Both halves are read rather than retyped: the architecture from the model's
own config.json and parameter count, the hardware from the running fleet.
Attention heads in place of KV heads is the classic slip and is worth a
factor of five on a model with grouped-query attention, so it is not a
number anyone should be entering.

Every candidate is listed, not one answer. With cards to spare you want the
full-precision fit; with one card free you want to know a quantised one
exists. Choosing between those is not the planner's job.

The tests are calibrated against files actually downloaded here — llama.cpp
reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the
model reproduces both — and they caught two errors in my own arithmetic.
The overhead was charged per rank from the whole model's weights rather
than each rank's share, and it reserved a tenth of a card that
usableFraction had already reserved. Between them, Llama-3.3-70B at bf16
across two 80 GB cards came out as not fitting, which is a configuration
people run every day. Slack reserved in two places, each unaware of the
other, reads as caution and behaves as a bug.

Pools are grouped by class and fabric together, because a replica may never
span two fabrics: eight free cards split across two of them are not eight
cards for a model that needs eight, and merging them would plan something
that starts, passes its health checks and hangs on its first all-reduce.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 07:46:53 +02:00

168 lines
5.1 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package modelplan
import (
"fmt"
"sort"
)
// Hardware is what one kind of node offers, as the fleet reports it.
type Hardware struct {
// Class is the GPU class, e.g. "h100-80g". Empty means a CPU-only node.
Class string
// GPUMemGB is one card's memory.
GPUMemGB int
// GPUsPerNode is how many cards a node of this kind has free.
GPUsPerNode int
// Nodes is how many such nodes are available.
Nodes int
// Fabric is what they are attached to, empty when they are not. A replica
// may never span two fabrics, so nodes on different ones are separate
// pools however identical their cards.
Fabric string
// HostMemGB is the machine's RAM, which is what a CPU profile needs.
HostMemGB int
}
// Fit is one way a model could run.
type Fit struct {
Precision Precision
Class string
TP int
Nodes int
GPUsPerNode int
Fabric string
// Context is the token pool this fit was checked against.
Context int
// PerGPUGB is what each card must hold, and Headroom what is left of it.
PerGPUGB float64
Headroom float64
// CPUOnly marks a fit that needs no GPU at all.
CPUOnly bool
}
// usableFraction is how much of a card a plan may assume it can have.
//
// Not all of it. A framework's allocator fragments, the driver reserves some,
// and a card planned to ninety-nine percent full is a card that fails to load
// on a bad day rather than a good one. vLLM's own default GPU utilisation is
// 0.90 and this matches it deliberately: a plan that disagrees with the engine
// it is planning for is worse than no plan.
const usableFraction = 0.90
// Fits returns every way this model could run on the given hardware, best
// first — fewest GPUs, then fewest nodes, then highest precision.
//
// Every candidate, not one answer, because the choice between them is not the
// planner's to make: a team with spare cards wants the full-precision fit, and
// a team with one card free wants to know a quantised one exists. Returning
// only the "best" would hide the option someone actually wants.
func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit {
if context <= 0 {
context = m.MaxContext
}
if len(precisions) == 0 {
precisions = []Precision{BF16, FP8, Q8GGUF, Q4GGUF}
}
var out []Fit
for _, p := range precisions {
d := m.Needs(p, context)
for _, h := range hw {
out = append(out, fitsOn(m, h, d, p, context)...)
}
}
sort.SliceStable(out, func(i, j int) bool {
a, b := out[i], out[j]
if ga, gb := a.TP, b.TP; ga != gb {
return ga < gb
}
if a.Nodes != b.Nodes {
return a.Nodes < b.Nodes
}
// Higher precision first among equals: the same hardware serving a
// better model is strictly the better answer.
return a.Precision.BytesPerParam > b.Precision.BytesPerParam
})
return out
}
func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
if h.Class == "" {
return cpuFit(h, d, p, context)
}
var out []Fit
budget := float64(h.GPUMemGB) * usableFraction
// Powers of two only. Tensor parallelism divides attention heads across
// ranks, and an engine that cannot divide them evenly refuses the profile —
// so offering tp=3 would produce a plan that fails at launch.
for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 {
nodes := 1
perNode := width
if width > h.GPUsPerNode {
perNode = h.GPUsPerNode
nodes = width / h.GPUsPerNode
}
if nodes > h.Nodes {
break
}
// A replica spanning nodes needs a fabric. Without one every node is
// its own island, and a multi-node replica would hang on its first
// all-reduce rather than fail to start.
if nodes > 1 && h.Fabric == "" {
continue
}
// The KV heads have to divide too, and they are far fewer than the
// attention heads under grouped-query attention: a model with 8 KV
// heads cannot be split sixteen ways.
if m.KVHeads > 0 && width > m.KVHeads {
break
}
per := GB(d.PerGPU(width))
if per > budget {
continue
}
out = append(out, Fit{
Precision: p, Class: h.Class, TP: width, Nodes: nodes,
GPUsPerNode: perNode, Fabric: h.Fabric, Context: context,
PerGPUGB: per, Headroom: budget - per,
})
}
return out
}
// cpuFit checks a node with no cards, where the whole working set lives in RAM.
func cpuFit(h Hardware, d Demand, p Precision, context int) []Fit {
if h.HostMemGB <= 0 {
return nil
}
// Less of the machine than of a card: the operating system, the page cache
// and everything else on the node want their share, and a CPU node is
// rarely dedicated the way a GPU node is.
budget := float64(h.HostMemGB) * 0.75
need := GB(d.Total())
if need > budget {
return nil
}
return []Fit{{
Precision: p, TP: 1, Nodes: 1, Context: context,
PerGPUGB: need, Headroom: budget - need, CPUOnly: true,
}}
}
// Describe renders a fit the way someone would say it out loud.
func (f Fit) Describe() string {
if f.CPUOnly {
return fmt.Sprintf("CPU, %s, %.1f GB of RAM", f.Precision.Name, f.PerGPUGB)
}
where := fmt.Sprintf("%d×%s", f.TP, f.Class)
if f.Nodes > 1 {
where = fmt.Sprintf("%d nodes × %d×%s on %s", f.Nodes, f.GPUsPerNode, f.Class, f.Fabric)
}
return fmt.Sprintf("%s, %s, %.1f GB a card (%.1f free)",
where, f.Precision.Name, f.PerGPUGB, f.Headroom)
}