Work out whether a model fits the cluster, and how to split it
llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct The arithmetic this replaces is the last thing between having GPUs and serving a model on them. Choosing tp, gpu_class and a context that will actually fit means knowing the layer count, the KV head count, what the weights weigh at a chosen quantisation and how much of a card the runtime keeps — and getting it wrong is not an error message, it is an out-of-memory twenty minutes into a load. I computed those by hand for every model on this cluster. Both halves are read rather than retyped: the architecture from the model's own config.json and parameter count, the hardware from the running fleet. Attention heads in place of KV heads is the classic slip and is worth a factor of five on a model with grouped-query attention, so it is not a number anyone should be entering. Every candidate is listed, not one answer. With cards to spare you want the full-precision fit; with one card free you want to know a quantised one exists. Choosing between those is not the planner's job. The tests are calibrated against files actually downloaded here — llama.cpp reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the model reproduces both — and they caught two errors in my own arithmetic. The overhead was charged per rank from the whole model's weights rather than each rank's share, and it reserved a tenth of a card that usableFraction had already reserved. Between them, Llama-3.3-70B at bf16 across two 80 GB cards came out as not fitting, which is a configuration people run every day. Slack reserved in two places, each unaware of the other, reads as caution and behaves as a bug. Pools are grouped by class and fabric together, because a replica may never span two fabrics: eight free cards split across two of them are not eight cards for a model that needs eight, and merging them would plan something that starts, passes its health checks and hangs on its first all-reduce. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ea4dd37f80
commit
300e298053
9 files changed
+1029
-16
No files matched your search
@@ -25,7 +25,7 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client))
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// newModelPlanCmd answers whether a model fits this cluster, and how.
|
||||
//
|
||||
// The arithmetic it replaces is the last thing standing between having GPUs and
|
||||
// serving a model on them. Deciding tp, gpu_class and a context that will
|
||||
// actually fit means knowing the model's layer count, its KV head count, what
|
||||
// its weights weigh at a chosen quantisation, and how much of a card the
|
||||
// runtime keeps — and getting it wrong is not an error message, it is an
|
||||
// out-of-memory twenty minutes into a load.
|
||||
//
|
||||
// It reads the model from its own repository and the hardware from the running
|
||||
// fleet, so neither half is a number somebody retyped.
|
||||
func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
context int
|
||||
params int64
|
||||
precs []string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "plan <huggingface-repo>",
|
||||
Short: "Work out whether a model fits this cluster, and how it should be split",
|
||||
Long: "Reads the model's architecture from its repository and the hardware from\n" +
|
||||
"the running fleet, then prints every way it could run — fewest GPUs\n" +
|
||||
"first — and a catalog entry for the best one.\n\n" +
|
||||
"Every candidate is listed rather than one answer, because the choice is\n" +
|
||||
"not the planner's to make: with cards to spare you want the\n" +
|
||||
"full-precision fit, and with one card free you want to know a quantised\n" +
|
||||
"one exists.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
repo := args[0]
|
||||
m, err := modelplan.FetchModel(cmd.Context(), repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if params > 0 {
|
||||
m.Params = params
|
||||
}
|
||||
if m.Params == 0 {
|
||||
return fmt.Errorf("%s does not publish a parameter count; pass --params", repo)
|
||||
}
|
||||
|
||||
var chosen []modelplan.Precision
|
||||
for _, name := range precs {
|
||||
p, err := modelplan.PrecisionByName(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chosen = append(chosen, p)
|
||||
}
|
||||
|
||||
var nodes []adminapi.NodeView
|
||||
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
hw := hardwareFrom(nodes)
|
||||
if len(hw) == 0 {
|
||||
return fmt.Errorf("no node is ready, so there is nothing to plan against")
|
||||
}
|
||||
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %.1fB parameters, %d layers, %d KV heads, trained to %d tokens\n",
|
||||
m.Name, float64(m.Params)/1e9, m.Layers, m.KVHeads, m.MaxContext)
|
||||
|
||||
want := context
|
||||
if want <= 0 {
|
||||
want = m.MaxContext
|
||||
}
|
||||
fmt.Fprintf(out, "Planned for a %d-token pool. %s a token of context.\n\n",
|
||||
want, humanBytes(m.KVBytesPerToken(modelplan.BF16)))
|
||||
|
||||
fits := modelplan.Fits(m, hw, want, chosen)
|
||||
if len(fits) == 0 {
|
||||
fmt.Fprintln(out, "It does not fit this cluster at any offered precision.")
|
||||
fmt.Fprintln(out, "\nWhat would change that:")
|
||||
fmt.Fprintln(out, " - a smaller context pool, with --context")
|
||||
fmt.Fprintln(out, " - a heavier quantisation, with --precision q4_k_m")
|
||||
fmt.Fprintln(out, " - more cards free; llmgw ctl status shows what is in use")
|
||||
return errDoesNotFit
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, " GPUS\tPRECISION\tPER CARD\tFREE\tWHERE")
|
||||
for _, f := range fits {
|
||||
where := f.Class
|
||||
if f.CPUOnly {
|
||||
where = "cpu"
|
||||
} else if f.Nodes > 1 {
|
||||
where = fmt.Sprintf("%s, %d nodes on %s", f.Class, f.Nodes, f.Fabric)
|
||||
}
|
||||
fmt.Fprintf(w, " %d\t%s\t%.1f GB\t%.1f GB\t%s\n",
|
||||
f.TP, f.Precision.Name, f.PerGPUGB, f.Headroom, where)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
fmt.Fprintf(out, "\nA catalog entry for the first of those:\n\n%s\n",
|
||||
catalogEntry(m, fits[0], repo, want))
|
||||
fmt.Fprintln(out, "Save it and add it with: llmgw ctl model add -f <file>")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
|
||||
c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one")
|
||||
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
|
||||
return c
|
||||
}
|
||||
|
||||
// errDoesNotFit gives a script something to test without parsing prose.
|
||||
var errDoesNotFit = fmt.Errorf("the model does not fit this cluster")
|
||||
|
||||
// hardwareFrom turns the fleet into pools a replica could actually occupy.
|
||||
//
|
||||
// Grouped by class and fabric together, because a replica may never span two
|
||||
// fabrics: eight free cards split across two fabrics are not eight cards for a
|
||||
// model that needs eight, and treating them as one pool would plan something
|
||||
// that hangs on its first all-reduce.
|
||||
func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||
type key struct{ class, fabric string }
|
||||
pools := map[key]*modelplan.Hardware{}
|
||||
|
||||
for _, n := range nodes {
|
||||
if n.State != "ready" {
|
||||
continue
|
||||
}
|
||||
fabric := ""
|
||||
if len(n.Fabrics) > 0 {
|
||||
fabric = n.Fabrics[0]
|
||||
}
|
||||
class := ""
|
||||
if len(n.Classes) > 0 {
|
||||
class = n.Classes[0]
|
||||
}
|
||||
k := key{class, fabric}
|
||||
if pools[k] == nil {
|
||||
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||
}
|
||||
p := pools[k]
|
||||
p.Nodes++
|
||||
// The widest a single node could contribute, not the total: a tensor
|
||||
// group takes the same number of cards from every node it spans.
|
||||
if n.GPUFree > p.GPUsPerNode {
|
||||
p.GPUsPerNode = n.GPUFree
|
||||
}
|
||||
if n.MemoryGB > p.HostMemGB {
|
||||
p.HostMemGB = n.MemoryGB
|
||||
}
|
||||
// Card memory from the class label, which carries it by construction:
|
||||
// "h100-80g" is eighty gigabytes and says so.
|
||||
if class != "" && p.GPUMemGB == 0 {
|
||||
p.GPUMemGB = memFromClass(class)
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]modelplan.Hardware, 0, len(pools))
|
||||
for _, p := range pools {
|
||||
if p.Class != "" && (p.GPUMemGB == 0 || p.GPUsPerNode == 0) {
|
||||
continue // a GPU pool with no free cards is not a pool
|
||||
}
|
||||
out = append(out, *p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// memFromClass reads the card size out of a class label such as "h100-80g".
|
||||
func memFromClass(class string) int {
|
||||
i := strings.LastIndex(class, "-")
|
||||
if i < 0 || !strings.HasSuffix(class, "g") {
|
||||
return 0
|
||||
}
|
||||
var gb int
|
||||
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
|
||||
return 0
|
||||
}
|
||||
return gb
|
||||
}
|
||||
|
||||
// catalogEntry renders a fit as the YAML that would serve it.
|
||||
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
|
||||
var b strings.Builder
|
||||
name := strings.ToLower(m.Name)
|
||||
fmt.Fprintf(&b, "name: %s\n", name)
|
||||
fmt.Fprintf(&b, "source: %s\n", source)
|
||||
fmt.Fprintf(&b, "routing_profile: chat\n")
|
||||
fmt.Fprintf(&b, "context_length: %d\n", context)
|
||||
fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n")
|
||||
fmt.Fprintf(&b, "profiles:\n")
|
||||
|
||||
switch {
|
||||
case f.CPUOnly:
|
||||
fmt.Fprintf(&b, " - name: cpu\n engine: llamacpp\n")
|
||||
fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2)
|
||||
default:
|
||||
fmt.Fprintf(&b, " - name: tp%d\n engine: vllm\n", f.TP)
|
||||
fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP)
|
||||
if f.Nodes > 1 {
|
||||
fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, " priority: 100\n weight: 1\n")
|
||||
// Measured rather than guessed is the rule everywhere else in this system,
|
||||
// and a planner should say so rather than pretend its estimate is one.
|
||||
fmt.Fprintf(&b, " startup_timeout: 10m\n")
|
||||
b.WriteString("\n# weight should come from `llmgw ctl bench` once it is running;\n")
|
||||
b.WriteString("# 1 is a placeholder, not a measurement.\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// humanBytes renders a per-token cache size, which is always small.
|
||||
func humanBytes(n int64) string {
|
||||
switch {
|
||||
case n >= 1<<20:
|
||||
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
||||
default:
|
||||
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
|
||||
}
|
||||
}
|
||||
@@ -59,3 +59,87 @@ func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) {
|
||||
// A replica may never span two fabrics, so eight free cards split across
|
||||
// two of them are not eight cards for a model that needs eight. Merging
|
||||
// them would plan something that starts, passes health checks and hangs on
|
||||
// its first all-reduce.
|
||||
nodes := []adminapi.NodeView{
|
||||
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||
{ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||
{ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4},
|
||||
}
|
||||
hw := hardwareFrom(nodes)
|
||||
if len(hw) != 2 {
|
||||
t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw)
|
||||
}
|
||||
for _, p := range hw {
|
||||
switch p.Fabric {
|
||||
case "ib-a":
|
||||
if p.Nodes != 2 {
|
||||
t.Errorf("ib-a has %d nodes, want 2", p.Nodes)
|
||||
}
|
||||
case "ib-b":
|
||||
if p.Nodes != 1 {
|
||||
t.Errorf("ib-b has %d nodes, want 1", p.Nodes)
|
||||
}
|
||||
default:
|
||||
t.Errorf("unexpected fabric %q", p.Fabric)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) {
|
||||
// Planning against a node that is draining or unreachable produces a fit
|
||||
// that cannot be placed, which is worse than saying nothing fits.
|
||||
nodes := []adminapi.NodeView{
|
||||
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
{ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
{ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||
}
|
||||
hw := hardwareFrom(nodes)
|
||||
if len(hw) != 1 || hw[0].Nodes != 1 {
|
||||
t.Fatalf("expected only the ready node, got %+v", hw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCardMemoryComesFromTheClassLabel(t *testing.T) {
|
||||
// The label carries it by construction — "h100-80g" is eighty gigabytes and
|
||||
// says so — which is why the planner needs no table of card models.
|
||||
for label, want := range map[string]int{
|
||||
"h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24,
|
||||
"nonsense": 0, "": 0, "h100-": 0,
|
||||
} {
|
||||
if got := memFromClass(label); got != want {
|
||||
t.Errorf("memFromClass(%q) = %d, want %d", label, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) {
|
||||
// How this cluster began: a node with no cards is not an empty pool, it is
|
||||
// a pool measured in RAM.
|
||||
hw := hardwareFrom([]adminapi.NodeView{
|
||||
{ID: "cpu1", State: "ready", MemoryGB: 31},
|
||||
})
|
||||
if len(hw) != 1 {
|
||||
t.Fatalf("the CPU node produced %d pools", len(hw))
|
||||
}
|
||||
if hw[0].Class != "" || hw[0].HostMemGB != 31 {
|
||||
t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
|
||||
// Fully allocated hardware is not capacity, and offering it would plan a
|
||||
// replica that cannot be placed.
|
||||
hw := hardwareFrom([]adminapi.NodeView{
|
||||
{ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8},
|
||||
})
|
||||
for _, p := range hw {
|
||||
if p.Class != "" {
|
||||
t.Errorf("a node with no free cards became a pool: %+v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Hardware is what one kind of node offers, as the fleet reports it.
|
||||
type Hardware struct {
|
||||
// Class is the GPU class, e.g. "h100-80g". Empty means a CPU-only node.
|
||||
Class string
|
||||
// GPUMemGB is one card's memory.
|
||||
GPUMemGB int
|
||||
// GPUsPerNode is how many cards a node of this kind has free.
|
||||
GPUsPerNode int
|
||||
// Nodes is how many such nodes are available.
|
||||
Nodes int
|
||||
// Fabric is what they are attached to, empty when they are not. A replica
|
||||
// may never span two fabrics, so nodes on different ones are separate
|
||||
// pools however identical their cards.
|
||||
Fabric string
|
||||
// HostMemGB is the machine's RAM, which is what a CPU profile needs.
|
||||
HostMemGB int
|
||||
}
|
||||
|
||||
// Fit is one way a model could run.
|
||||
type Fit struct {
|
||||
Precision Precision
|
||||
Class string
|
||||
TP int
|
||||
Nodes int
|
||||
GPUsPerNode int
|
||||
Fabric string
|
||||
// Context is the token pool this fit was checked against.
|
||||
Context int
|
||||
// PerGPUGB is what each card must hold, and Headroom what is left of it.
|
||||
PerGPUGB float64
|
||||
Headroom float64
|
||||
// CPUOnly marks a fit that needs no GPU at all.
|
||||
CPUOnly bool
|
||||
}
|
||||
|
||||
// usableFraction is how much of a card a plan may assume it can have.
|
||||
//
|
||||
// Not all of it. A framework's allocator fragments, the driver reserves some,
|
||||
// and a card planned to ninety-nine percent full is a card that fails to load
|
||||
// on a bad day rather than a good one. vLLM's own default GPU utilisation is
|
||||
// 0.90 and this matches it deliberately: a plan that disagrees with the engine
|
||||
// it is planning for is worse than no plan.
|
||||
const usableFraction = 0.90
|
||||
|
||||
// Fits returns every way this model could run on the given hardware, best
|
||||
// first — fewest GPUs, then fewest nodes, then highest precision.
|
||||
//
|
||||
// Every candidate, not one answer, because the choice between them is not the
|
||||
// planner's to make: a team with spare cards wants the full-precision fit, and
|
||||
// a team with one card free wants to know a quantised one exists. Returning
|
||||
// only the "best" would hide the option someone actually wants.
|
||||
func Fits(m Model, hw []Hardware, context int, precisions []Precision) []Fit {
|
||||
if context <= 0 {
|
||||
context = m.MaxContext
|
||||
}
|
||||
if len(precisions) == 0 {
|
||||
precisions = []Precision{BF16, FP8, Q8GGUF, Q4GGUF}
|
||||
}
|
||||
|
||||
var out []Fit
|
||||
for _, p := range precisions {
|
||||
d := m.Needs(p, context)
|
||||
for _, h := range hw {
|
||||
out = append(out, fitsOn(m, h, d, p, context)...)
|
||||
}
|
||||
}
|
||||
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
if ga, gb := a.TP, b.TP; ga != gb {
|
||||
return ga < gb
|
||||
}
|
||||
if a.Nodes != b.Nodes {
|
||||
return a.Nodes < b.Nodes
|
||||
}
|
||||
// Higher precision first among equals: the same hardware serving a
|
||||
// better model is strictly the better answer.
|
||||
return a.Precision.BytesPerParam > b.Precision.BytesPerParam
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func fitsOn(m Model, h Hardware, d Demand, p Precision, context int) []Fit {
|
||||
if h.Class == "" {
|
||||
return cpuFit(h, d, p, context)
|
||||
}
|
||||
|
||||
var out []Fit
|
||||
budget := float64(h.GPUMemGB) * usableFraction
|
||||
|
||||
// Powers of two only. Tensor parallelism divides attention heads across
|
||||
// ranks, and an engine that cannot divide them evenly refuses the profile —
|
||||
// so offering tp=3 would produce a plan that fails at launch.
|
||||
for width := 1; width <= h.GPUsPerNode*h.Nodes; width *= 2 {
|
||||
nodes := 1
|
||||
perNode := width
|
||||
if width > h.GPUsPerNode {
|
||||
perNode = h.GPUsPerNode
|
||||
nodes = width / h.GPUsPerNode
|
||||
}
|
||||
if nodes > h.Nodes {
|
||||
break
|
||||
}
|
||||
// A replica spanning nodes needs a fabric. Without one every node is
|
||||
// its own island, and a multi-node replica would hang on its first
|
||||
// all-reduce rather than fail to start.
|
||||
if nodes > 1 && h.Fabric == "" {
|
||||
continue
|
||||
}
|
||||
// The KV heads have to divide too, and they are far fewer than the
|
||||
// attention heads under grouped-query attention: a model with 8 KV
|
||||
// heads cannot be split sixteen ways.
|
||||
if m.KVHeads > 0 && width > m.KVHeads {
|
||||
break
|
||||
}
|
||||
|
||||
per := GB(d.PerGPU(width))
|
||||
if per > budget {
|
||||
continue
|
||||
}
|
||||
out = append(out, Fit{
|
||||
Precision: p, Class: h.Class, TP: width, Nodes: nodes,
|
||||
GPUsPerNode: perNode, Fabric: h.Fabric, Context: context,
|
||||
PerGPUGB: per, Headroom: budget - per,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cpuFit checks a node with no cards, where the whole working set lives in RAM.
|
||||
func cpuFit(h Hardware, d Demand, p Precision, context int) []Fit {
|
||||
if h.HostMemGB <= 0 {
|
||||
return nil
|
||||
}
|
||||
// Less of the machine than of a card: the operating system, the page cache
|
||||
// and everything else on the node want their share, and a CPU node is
|
||||
// rarely dedicated the way a GPU node is.
|
||||
budget := float64(h.HostMemGB) * 0.75
|
||||
need := GB(d.Total())
|
||||
if need > budget {
|
||||
return nil
|
||||
}
|
||||
return []Fit{{
|
||||
Precision: p, TP: 1, Nodes: 1, Context: context,
|
||||
PerGPUGB: need, Headroom: budget - need, CPUOnly: true,
|
||||
}}
|
||||
}
|
||||
|
||||
// Describe renders a fit the way someone would say it out loud.
|
||||
func (f Fit) Describe() string {
|
||||
if f.CPUOnly {
|
||||
return fmt.Sprintf("CPU, %s, %.1f GB of RAM", f.Precision.Name, f.PerGPUGB)
|
||||
}
|
||||
where := fmt.Sprintf("%d×%s", f.TP, f.Class)
|
||||
if f.Nodes > 1 {
|
||||
where = fmt.Sprintf("%d nodes × %d×%s on %s", f.Nodes, f.GPUsPerNode, f.Class, f.Fabric)
|
||||
}
|
||||
return fmt.Sprintf("%s, %s, %.1f GB a card (%.1f free)",
|
||||
where, f.Precision.Name, f.PerGPUGB, f.Headroom)
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func h100Node(nodes, gpus int, fabric string) Hardware {
|
||||
return Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: gpus, Nodes: nodes, Fabric: fabric}
|
||||
}
|
||||
|
||||
func TestASmallModelFitsOneCard(t *testing.T) {
|
||||
fits := Fits(coder3B, []Hardware{h100Node(4, 8, "ib-a")}, 32768, []Precision{BF16})
|
||||
if len(fits) == 0 {
|
||||
t.Fatal("a 3B was reported as not fitting an H100")
|
||||
}
|
||||
// Best first, and the best answer for a small model is one card.
|
||||
if fits[0].TP != 1 {
|
||||
t.Errorf("best fit uses %d GPUs; a 3B needs one", fits[0].TP)
|
||||
}
|
||||
}
|
||||
|
||||
func TestA70BNeedsTwoCardsAtFullPrecision(t *testing.T) {
|
||||
// The configuration people run every day, and the one an earlier overhead
|
||||
// model wrongly called impossible by reserving slack twice.
|
||||
fits := Fits(llama70B, []Hardware{h100Node(4, 8, "ib-a")}, 8192, []Precision{BF16})
|
||||
if len(fits) == 0 {
|
||||
t.Fatal("70B bf16 reported as not fitting eight H100s")
|
||||
}
|
||||
if fits[0].TP != 2 {
|
||||
t.Errorf("best fit is tp=%d; 70B at bf16 on two 80 GB cards is routine", fits[0].TP)
|
||||
}
|
||||
for _, f := range fits {
|
||||
if f.TP == 1 {
|
||||
t.Errorf("70B bf16 reported as fitting a single card: %s", f.Describe())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextChangesHowManyCardsAModelNeeds(t *testing.T) {
|
||||
// The fact that makes this worth computing rather than looking up. The same
|
||||
// 70B at the same precision needs two cards for an 8k pool and four for
|
||||
// 32k, because the KV cache is what grows — and "70B needs two H100s" is
|
||||
// repeated everywhere without the context it was true for.
|
||||
hw := []Hardware{h100Node(4, 8, "ib-a")}
|
||||
short := Fits(llama70B, hw, 8192, []Precision{BF16})
|
||||
long := Fits(llama70B, hw, 32768, []Precision{BF16})
|
||||
if len(short) == 0 || len(long) == 0 {
|
||||
t.Fatal("expected fits at both context sizes")
|
||||
}
|
||||
if long[0].TP <= short[0].TP {
|
||||
t.Errorf("tp at 32k is %d and at 8k is %d; a larger context should need at least as much",
|
||||
long[0].TP, short[0].TP)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuantisationOpensUpSmallerHardware(t *testing.T) {
|
||||
// The answer someone with one spare card needs: not "no", but "not at this
|
||||
// precision".
|
||||
one := []Hardware{h100Node(1, 1, "")}
|
||||
if full := Fits(llama70B, one, 8192, []Precision{BF16}); len(full) != 0 {
|
||||
t.Errorf("70B bf16 fitted a single card: %s", full[0].Describe())
|
||||
}
|
||||
if quant := Fits(llama70B, one, 8192, []Precision{Q4GGUF}); len(quant) == 0 {
|
||||
t.Error("70B at q4 did not fit a single 80 GB card, which it should")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATensorGroupNeverSpansFabrics(t *testing.T) {
|
||||
// Two nodes with no fabric are two islands. A multi-node replica placed
|
||||
// across them would start, pass health checks and hang on its first
|
||||
// all-reduce, which is worse than refusing to plan it.
|
||||
noFabric := Hardware{Class: "h100-80g", GPUMemGB: 80, GPUsPerNode: 2, Nodes: 4}
|
||||
for _, f := range Fits(llama70B, []Hardware{noFabric}, 8192, []Precision{BF16}) {
|
||||
if f.Nodes > 1 {
|
||||
t.Errorf("planned %d nodes with no fabric: %s", f.Nodes, f.Describe())
|
||||
}
|
||||
}
|
||||
withFabric := noFabric
|
||||
withFabric.Fabric = "ib-a"
|
||||
var spanning bool
|
||||
for _, f := range Fits(llama70B, []Hardware{withFabric}, 8192, []Precision{BF16}) {
|
||||
if f.Nodes > 1 {
|
||||
spanning = true
|
||||
}
|
||||
}
|
||||
if !spanning {
|
||||
t.Error("a fabric was available and no multi-node fit was offered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTensorWidthCannotExceedTheKVHeads(t *testing.T) {
|
||||
// Grouped-query attention leaves far fewer KV heads than attention heads,
|
||||
// and an engine refuses a width that cannot divide them. Offering tp=16 for
|
||||
// a model with 8 KV heads is a plan that fails at launch.
|
||||
for _, f := range Fits(llama70B, []Hardware{h100Node(8, 8, "ib-a")}, 8192, []Precision{Q4GGUF}) {
|
||||
if f.TP > llama70B.KVHeads {
|
||||
t.Errorf("tp=%d exceeds the model's %d KV heads", f.TP, llama70B.KVHeads)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyPowersOfTwoAreOffered(t *testing.T) {
|
||||
for _, f := range Fits(llama70B, []Hardware{h100Node(4, 6, "ib-a")}, 8192, []Precision{Q4GGUF}) {
|
||||
if f.TP&(f.TP-1) != 0 {
|
||||
t.Errorf("tp=%d is not a power of two", f.TP)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACardIsNeverPlannedToTheBrim(t *testing.T) {
|
||||
// A plan that fills a card completely is one that loads on a good day.
|
||||
// This matches vLLM's own 0.90 default deliberately: a planner that
|
||||
// disagrees with the engine it plans for is worse than none.
|
||||
for _, f := range Fits(coder14B, []Hardware{h100Node(1, 8, "")}, 32768, []Precision{BF16}) {
|
||||
if f.PerGPUGB > 80*usableFraction+0.01 {
|
||||
t.Errorf("planned %.1f GB on an 80 GB card: %s", f.PerGPUGB, f.Describe())
|
||||
}
|
||||
if f.Headroom < 0 {
|
||||
t.Errorf("negative headroom: %s", f.Describe())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACPUNodeCanServeASmallModel(t *testing.T) {
|
||||
// How this cluster began, and a reasonable way to try llmgw before
|
||||
// committing cards to it.
|
||||
cpu := Hardware{HostMemGB: 32}
|
||||
fits := Fits(coder3B, []Hardware{cpu}, 32768, []Precision{Q4GGUF})
|
||||
if len(fits) == 0 {
|
||||
t.Fatal("a 3B at q4 did not fit a 32 GB CPU node")
|
||||
}
|
||||
if !fits[0].CPUOnly {
|
||||
t.Error("the fit is not marked CPU-only")
|
||||
}
|
||||
if !strings.Contains(fits[0].Describe(), "CPU") {
|
||||
t.Errorf("the description does not say it is CPU: %q", fits[0].Describe())
|
||||
}
|
||||
// And a 70B does not, whatever the quantisation.
|
||||
if big := Fits(llama70B, []Hardware{cpu}, 8192, []Precision{Q4GGUF}); len(big) != 0 {
|
||||
t.Errorf("a 70B was reported as fitting 32 GB of RAM: %s", big[0].Describe())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingFitsIsAnEmptyAnswerNotAWrongOne(t *testing.T) {
|
||||
tiny := Hardware{Class: "t4-16g", GPUMemGB: 16, GPUsPerNode: 1, Nodes: 1}
|
||||
if fits := Fits(llama70B, []Hardware{tiny}, 32768, []Precision{BF16}); len(fits) != 0 {
|
||||
t.Errorf("70B bf16 fitted a single 16 GB card: %s", fits[0].Describe())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FetchModel reads a model's architecture from its repository.
|
||||
//
|
||||
// Read rather than asked for, because every field here is one somebody would
|
||||
// otherwise have to look up and retype: the layer count, the KV head count, the
|
||||
// head dimension and the parameter total. Those are exactly the numbers a
|
||||
// person gets subtly wrong — attention heads instead of KV heads being the
|
||||
// classic, and worth a factor of five on a model with grouped-query attention.
|
||||
//
|
||||
// Two documents: config.json for the architecture, and the repository index for
|
||||
// the parameter count, which is the one number config.json does not carry.
|
||||
func FetchModel(ctx context.Context, repo string) (Model, error) {
|
||||
base := "https://huggingface.co"
|
||||
cfg, err := fetchConfig(ctx, base, repo)
|
||||
if err != nil {
|
||||
return Model{}, err
|
||||
}
|
||||
m, err := cfg.toModel(repo)
|
||||
if err != nil {
|
||||
return Model{}, err
|
||||
}
|
||||
// Best effort. A repository that does not publish its parameter count is
|
||||
// still usable — the caller can supply one — and failing the whole lookup
|
||||
// over it would be worse than saying which number is missing.
|
||||
if n, err := fetchParamCount(ctx, base, repo); err == nil && n > 0 {
|
||||
m.Params = n
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type hfConfig struct {
|
||||
NumHiddenLayers int `json:"num_hidden_layers"`
|
||||
NumAttentionHeads int `json:"num_attention_heads"`
|
||||
NumKeyValueHeads int `json:"num_key_value_heads"`
|
||||
HiddenSize int `json:"hidden_size"`
|
||||
HeadDim int `json:"head_dim"`
|
||||
MaxPositions int `json:"max_position_embeddings"`
|
||||
ModelType string `json:"model_type"`
|
||||
}
|
||||
|
||||
func (c hfConfig) toModel(repo string) (Model, error) {
|
||||
if c.NumHiddenLayers == 0 {
|
||||
return Model{}, fmt.Errorf("%s: config.json has no num_hidden_layers; is it a text model?", repo)
|
||||
}
|
||||
m := Model{
|
||||
Name: repo[strings.LastIndex(repo, "/")+1:],
|
||||
Layers: c.NumHiddenLayers,
|
||||
MaxContext: c.MaxPositions,
|
||||
}
|
||||
|
||||
// Grouped-query attention: when the KV head count is absent the model has
|
||||
// as many as it has attention heads, which is multi-head attention and the
|
||||
// older, more expensive arrangement.
|
||||
m.KVHeads = c.NumKeyValueHeads
|
||||
if m.KVHeads == 0 {
|
||||
m.KVHeads = c.NumAttentionHeads
|
||||
}
|
||||
|
||||
// head_dim is explicit in newer configs and derived in older ones. Derived
|
||||
// from the attention heads, never the KV heads: the hidden size is split
|
||||
// across the query heads.
|
||||
m.HeadDim = c.HeadDim
|
||||
if m.HeadDim == 0 {
|
||||
if c.NumAttentionHeads == 0 {
|
||||
return Model{}, fmt.Errorf("%s: config.json gives neither head_dim nor num_attention_heads", repo)
|
||||
}
|
||||
m.HeadDim = c.HiddenSize / c.NumAttentionHeads
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func fetchConfig(ctx context.Context, base, repo string) (hfConfig, error) {
|
||||
var c hfConfig
|
||||
body, err := get(ctx, base+"/"+repo+"/resolve/main/config.json")
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("reading %s: %w", repo, err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &c); err != nil {
|
||||
return c, fmt.Errorf("%s: config.json did not parse: %w", repo, err)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fetchParamCount reads the parameter total from the repository's own index.
|
||||
//
|
||||
// Taken from the safetensors index rather than estimated from the architecture,
|
||||
// because an estimate is wrong by whatever the embeddings and the tied weights
|
||||
// come to, and that error lands directly on the weights figure — the largest
|
||||
// term in the whole calculation.
|
||||
func fetchParamCount(ctx context.Context, base, repo string) (int64, error) {
|
||||
body, err := get(ctx, base+"/api/models/"+repo)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var info struct {
|
||||
Safetensors struct {
|
||||
Total int64 `json:"total"`
|
||||
} `json:"safetensors"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &info); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.Safetensors.Total, nil
|
||||
}
|
||||
|
||||
func get(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
// A private or gated repository is the common case here, and "404" on
|
||||
// its own sends people looking for a typo they did not make.
|
||||
return nil, fmt.Errorf("%s returned %d; a private or gated repository needs to be downloaded by hand",
|
||||
url, resp.StatusCode)
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package modelplan
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestKVHeadsFallBackToAttentionHeads(t *testing.T) {
|
||||
// A config without num_key_value_heads is multi-head attention, where every
|
||||
// attention head keeps its own cache. Defaulting to zero instead would make
|
||||
// the KV cache vanish from the calculation entirely.
|
||||
m, err := hfConfig{
|
||||
NumHiddenLayers: 32, NumAttentionHeads: 32, HiddenSize: 4096,
|
||||
MaxPositions: 4096,
|
||||
}.toModel("org/old-model")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.KVHeads != 32 {
|
||||
t.Errorf("KVHeads = %d, want the attention head count", m.KVHeads)
|
||||
}
|
||||
if m.KVBytesPerToken(BF16) == 0 {
|
||||
t.Error("the KV cache came out as nothing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadDimIsDerivedFromAttentionHeadsNotKVHeads(t *testing.T) {
|
||||
// The hidden size is split across the query heads. Dividing by the KV heads
|
||||
// instead inflates head_dim by the grouping factor — eight, on a typical
|
||||
// model — and the error lands straight on the cache figure.
|
||||
m, err := hfConfig{
|
||||
NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8,
|
||||
HiddenSize: 5120, MaxPositions: 32768,
|
||||
}.toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.HeadDim != 128 {
|
||||
t.Errorf("HeadDim = %d, want 5120/40 = 128", m.HeadDim)
|
||||
}
|
||||
// Which reproduces the figure measured on this cluster.
|
||||
if got := m.KVBytesPerToken(BF16); got != 192*1024 {
|
||||
t.Errorf("KV/token = %d, want %d", got, 192*1024)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExplicitHeadDimWins(t *testing.T) {
|
||||
// Newer configs state it, and some models do not have hidden_size divisible
|
||||
// by the head count — deriving it there is simply wrong.
|
||||
m, err := hfConfig{
|
||||
NumHiddenLayers: 32, NumAttentionHeads: 32, NumKeyValueHeads: 8,
|
||||
HiddenSize: 4096, HeadDim: 256, MaxPositions: 8192,
|
||||
}.toModel("org/model")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.HeadDim != 256 {
|
||||
t.Errorf("HeadDim = %d; the explicit value should win over 4096/32", m.HeadDim)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConfigWithoutLayersIsRefusedClearly(t *testing.T) {
|
||||
// Pointing this at an embedding or vision repository is an easy mistake,
|
||||
// and a plan built from zeros would be confidently meaningless.
|
||||
_, err := hfConfig{MaxPositions: 512}.toModel("org/not-a-text-model")
|
||||
if err == nil {
|
||||
t.Fatal("a config with no layers was accepted")
|
||||
}
|
||||
if !contains(err.Error(), "num_hidden_layers") {
|
||||
t.Errorf("the error does not name what is missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) {
|
||||
m, _ := hfConfig{NumHiddenLayers: 1, NumAttentionHeads: 1, HiddenSize: 64}.
|
||||
toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
|
||||
if m.Name != "Qwen2.5-Coder-14B-Instruct" {
|
||||
t.Errorf("Name = %q", m.Name)
|
||||
}
|
||||
}
|
||||
+28
-15
@@ -63,24 +63,33 @@ func PrecisionByName(s string) (Precision, error) {
|
||||
|
||||
// Demand is what one replica needs, in bytes.
|
||||
type Demand struct {
|
||||
Weights int64
|
||||
KVCache int64
|
||||
Weights int64
|
||||
KVCache int64
|
||||
// Overhead is what the runtime costs on a single GPU. Across a tensor
|
||||
// parallel group it is recomputed from each rank's share — see PerGPU.
|
||||
Overhead int64
|
||||
}
|
||||
|
||||
// Total is everything one replica needs across all its GPUs.
|
||||
func (d Demand) Total() int64 { return d.Weights + d.KVCache + d.Overhead }
|
||||
|
||||
// PerGPU divides the demand over a replica's GPUs.
|
||||
// PerGPU is what one GPU of a replica must hold.
|
||||
//
|
||||
// Tensor parallelism splits both the weights and the KV cache, so both divide.
|
||||
// The overhead does not: each process pays its own runtime, CUDA context and
|
||||
// activation buffers, so it is charged per GPU rather than shared.
|
||||
// Tensor parallelism splits the weights and the KV cache, so both divide. The
|
||||
// overhead does not divide — each process pays its own CUDA context and
|
||||
// allocator — but neither is it constant: activations scale with the shard the
|
||||
// rank actually holds, so it is recomputed from the divided weights.
|
||||
//
|
||||
// Charging the whole model's overhead to every rank was the first version of
|
||||
// this, and it was badly wrong in exactly the case that matters. Llama-3.3-70B
|
||||
// at bf16 across two 80 GB cards came out at 85.9 GB a card and was reported as
|
||||
// not fitting, which is a configuration people run every day.
|
||||
func (d Demand) PerGPU(gpus int) int64 {
|
||||
if gpus <= 0 {
|
||||
gpus = 1
|
||||
}
|
||||
return (d.Weights+d.KVCache)/int64(gpus) + d.Overhead
|
||||
n := int64(gpus)
|
||||
return (d.Weights+d.KVCache)/n + overheadBytes(d.Weights/n)
|
||||
}
|
||||
|
||||
// KVBytesPerToken is what one token of context costs, for the whole model.
|
||||
@@ -108,17 +117,21 @@ func (m Model) Needs(p Precision, context int) Demand {
|
||||
}
|
||||
}
|
||||
|
||||
// overheadBytes is what the runtime costs beyond weights and cache.
|
||||
// overheadBytes is what the runtime costs beyond weights and cache, per GPU.
|
||||
//
|
||||
// A flat allowance plus a fraction of the weights: the flat part is the CUDA
|
||||
// context, the framework and its allocator, which a small model pays as much as
|
||||
// a large one; the fraction stands in for activations and fragmentation, which
|
||||
// do scale. Deliberately generous. A plan that says a model fits and is wrong
|
||||
// costs a twenty-minute load and an OOM; one that says it does not quite fit
|
||||
// costs a smaller quantisation.
|
||||
// A flat allowance for the CUDA context, the framework and its allocator —
|
||||
// which a small model pays as much as a large one — plus a small fraction of
|
||||
// the shard for activations, which do scale with it.
|
||||
//
|
||||
// The fraction is two percent rather than the ten it started at, because the
|
||||
// ten was counted twice. usableFraction already holds back a tenth of every
|
||||
// card for exactly this, so charging another tenth on top made a plan reject
|
||||
// Llama-3.3-70B at bf16 across two 80 GB cards — a configuration people run
|
||||
// every day. Reserving slack in two places, each unaware of the other, reads as
|
||||
// caution and behaves as a bug.
|
||||
func overheadBytes(weights int64) int64 {
|
||||
const flat = 2 << 30 // 2 GiB
|
||||
return flat + weights/10
|
||||
return flat + weights/50
|
||||
}
|
||||
|
||||
// GB renders bytes the way people discuss hardware.
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Real models, with the figures measured on this cluster rather than derived
|
||||
// from a formula — the point of testing arithmetic is that it agrees with the
|
||||
// world, and a test built from the same formula would agree with itself.
|
||||
var (
|
||||
coder3B = Model{
|
||||
Name: "Qwen2.5-Coder-3B", Params: 3_090_000_000,
|
||||
Layers: 36, KVHeads: 2, HeadDim: 128, MaxContext: 32768,
|
||||
}
|
||||
coder14B = Model{
|
||||
Name: "Qwen2.5-Coder-14B", Params: 14_770_000_000,
|
||||
Layers: 48, KVHeads: 8, HeadDim: 128, MaxContext: 32768,
|
||||
}
|
||||
llama70B = Model{
|
||||
Name: "Llama-3.3-70B", Params: 70_600_000_000,
|
||||
Layers: 80, KVHeads: 8, HeadDim: 128, MaxContext: 131072,
|
||||
}
|
||||
)
|
||||
|
||||
func closeTo(t *testing.T, what string, got, want, tolerance float64) {
|
||||
t.Helper()
|
||||
if math.Abs(got-want) > tolerance {
|
||||
t.Errorf("%s = %.2f, want %.2f (±%.2f)", what, got, want, tolerance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVCachePerTokenMatchesTheArchitecture(t *testing.T) {
|
||||
// 192 KiB a token for the 14B is the number that made sense of what this
|
||||
// cluster measured: 8192 of context across four llama.cpp slots is 2048
|
||||
// each, and six gigabytes of cache at the model's full 32k.
|
||||
if got := coder14B.KVBytesPerToken(BF16); got != 192*1024 {
|
||||
t.Errorf("14B KV/token = %d bytes, want %d", got, 192*1024)
|
||||
}
|
||||
// Grouped-query attention is why this is KVHeads and not the attention
|
||||
// head count. The 14B has 40 attention heads and 8 KV heads, and paying for
|
||||
// 40 would overstate the cache fivefold.
|
||||
if got := coder3B.KVBytesPerToken(BF16); got != 36*1024 {
|
||||
t.Errorf("3B KV/token = %d bytes, want %d", got, 36*1024)
|
||||
}
|
||||
if got := llama70B.KVBytesPerToken(BF16); got != 320*1024 {
|
||||
t.Errorf("70B KV/token = %d bytes, want %d", got, 320*1024)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuantisedWeightsMatchTheFilesOnDisk(t *testing.T) {
|
||||
// Calibrated against the GGUFs downloaded to this cluster, so the
|
||||
// bytes-per-parameter figures are measurements rather than folklore.
|
||||
// In GiB, which is what llama-bench itself reported for these files —
|
||||
// "14.62 GiB" and "8.37 GiB". The download sizes quoted in decimal GB are
|
||||
// the same bytes in different units, and mixing the two is how a plan ends
|
||||
// up seven percent optimistic.
|
||||
for _, tc := range []struct {
|
||||
p Precision
|
||||
want float64 // GiB, as llama.cpp accounts for it
|
||||
}{
|
||||
{Q8GGUF, 14.62},
|
||||
{Q4GGUF, 8.37},
|
||||
} {
|
||||
got := GB(coder14B.Needs(tc.p, 0).Weights)
|
||||
closeTo(t, "14B "+tc.p.Name+" weights", got, tc.want, 0.35)
|
||||
}
|
||||
// And full precision, which is two bytes a parameter by definition.
|
||||
closeTo(t, "14B bf16 weights", GB(coder14B.Needs(BF16, 0).Weights), 27.5, 0.6)
|
||||
}
|
||||
|
||||
func TestContextDominatesAtLongWindows(t *testing.T) {
|
||||
// The reason KV cache is worth computing rather than hand-waving: at 128k
|
||||
// of pooled context a 70B's cache is a substantial fraction of its weights,
|
||||
// and a plan that ignores it fits a model that then will not load.
|
||||
short := llama70B.Needs(BF16, 4096)
|
||||
long := llama70B.Needs(BF16, 131072)
|
||||
if short.Weights != long.Weights {
|
||||
t.Fatal("weights should not depend on context")
|
||||
}
|
||||
if GB(long.KVCache) < 35 {
|
||||
t.Errorf("70B KV at 128k = %.1f GB, expected tens of GB", GB(long.KVCache))
|
||||
}
|
||||
if long.KVCache <= short.KVCache*8 {
|
||||
t.Error("KV cache should scale with the context pool")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTensorParallelismDividesWeightsAndCacheButNotOverhead(t *testing.T) {
|
||||
// Each rank holds its share of the weights and of the KV heads, so both
|
||||
// divide. The runtime does not: every process pays its own CUDA context and
|
||||
// activation buffers, which is why eight-way parallelism does not make a
|
||||
// model eight times cheaper per card.
|
||||
d := llama70B.Needs(BF16, 32768)
|
||||
|
||||
one := d.PerGPU(1)
|
||||
eight := d.PerGPU(8)
|
||||
if eight >= one {
|
||||
t.Fatal("splitting across GPUs should reduce the per-GPU demand")
|
||||
}
|
||||
// Strictly more than an eighth, because the overhead is not shared.
|
||||
if eight <= one/8 {
|
||||
t.Errorf("per-GPU at tp=8 is %d, at or below a clean eighth of %d: overhead is being divided",
|
||||
eight, one)
|
||||
}
|
||||
if d.PerGPU(0) != d.PerGPU(1) {
|
||||
t.Error("a nonsensical GPU count should be treated as one, not divided by zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestA70BNeedsMoreThanOneCard(t *testing.T) {
|
||||
// The question someone with hardware actually asks, and the answer that is
|
||||
// easy to get wrong by hand: 70B at bf16 does not fit an 80 GB card, and
|
||||
// does fit two.
|
||||
d := llama70B.Needs(BF16, 32768)
|
||||
const h100 = 80.0
|
||||
|
||||
if GB(d.PerGPU(1)) <= h100 {
|
||||
t.Errorf("70B bf16 reported as fitting one 80 GB card (%.1f GB)", GB(d.PerGPU(1)))
|
||||
}
|
||||
if GB(d.PerGPU(2)) > h100 {
|
||||
t.Errorf("70B bf16 across two cards is %.1f GB each, which should fit 80", GB(d.PerGPU(2)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverheadIsChargedEvenWithNoContext(t *testing.T) {
|
||||
// A plan that forgets the runtime fits a model into exactly its weights and
|
||||
// then watches it fail to load.
|
||||
d := coder3B.Needs(BF16, 0)
|
||||
if d.Overhead <= 0 {
|
||||
t.Error("no overhead was charged")
|
||||
}
|
||||
if d.Total() <= d.Weights {
|
||||
t.Error("the total is not more than the weights alone")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrecisionByNameRefusesWhatItDoesNotKnow(t *testing.T) {
|
||||
if _, err := PrecisionByName("bf16"); err != nil {
|
||||
t.Errorf("bf16 was refused: %v", err)
|
||||
}
|
||||
_, err := PrecisionByName("fp3")
|
||||
if err == nil {
|
||||
t.Fatal("an unknown precision was accepted")
|
||||
}
|
||||
// The error lists what would work, because the caller mistyped something
|
||||
// and a bare refusal makes them go looking.
|
||||
if got := err.Error(); !contains(got, "q4_k_m") || !contains(got, "bf16") {
|
||||
t.Errorf("the error does not say what is known: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user