Ilker ManapandClaude Opus 5
300e298053
llmgw ctl model plan Qwen/Qwen2.5-Coder-32B-Instruct The arithmetic this replaces is the last thing between having GPUs and serving a model on them. Choosing tp, gpu_class and a context that will actually fit means knowing the layer count, the KV head count, what the weights weigh at a chosen quantisation and how much of a card the runtime keeps — and getting it wrong is not an error message, it is an out-of-memory twenty minutes into a load. I computed those by hand for every model on this cluster. Both halves are read rather than retyped: the architecture from the model's own config.json and parameter count, the hardware from the running fleet. Attention heads in place of KV heads is the classic slip and is worth a factor of five on a model with grouped-query attention, so it is not a number anyone should be entering. Every candidate is listed, not one answer. With cards to spare you want the full-precision fit; with one card free you want to know a quantised one exists. Choosing between those is not the planner's job. The tests are calibrated against files actually downloaded here — llama.cpp reported 14.62 GiB and 8.37 GiB for the 14B at q8_0 and q4_K_M, and the model reproduces both — and they caught two errors in my own arithmetic. The overhead was charged per rank from the whole model's weights rather than each rank's share, and it reserved a tenth of a card that usableFraction had already reserved. Between them, Llama-3.3-70B at bf16 across two 80 GB cards came out as not fitting, which is a configuration people run every day. Slack reserved in two places, each unaware of the other, reads as caution and behaves as a bug. Pools are grouped by class and fabric together, because a replica may never span two fabrics: eight free cards split across two of them are not eight cards for a model that needs eight, and merging them would plan something that starts, passes its health checks and hangs on its first all-reduce. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
78 lines
2.6 KiB
Go
78 lines
2.6 KiB
Go
package modelplan
|
|
|
|
import "testing"
|
|
|
|
func TestKVHeadsFallBackToAttentionHeads(t *testing.T) {
|
|
// A config without num_key_value_heads is multi-head attention, where every
|
|
// attention head keeps its own cache. Defaulting to zero instead would make
|
|
// the KV cache vanish from the calculation entirely.
|
|
m, err := hfConfig{
|
|
NumHiddenLayers: 32, NumAttentionHeads: 32, HiddenSize: 4096,
|
|
MaxPositions: 4096,
|
|
}.toModel("org/old-model")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.KVHeads != 32 {
|
|
t.Errorf("KVHeads = %d, want the attention head count", m.KVHeads)
|
|
}
|
|
if m.KVBytesPerToken(BF16) == 0 {
|
|
t.Error("the KV cache came out as nothing")
|
|
}
|
|
}
|
|
|
|
func TestHeadDimIsDerivedFromAttentionHeadsNotKVHeads(t *testing.T) {
|
|
// The hidden size is split across the query heads. Dividing by the KV heads
|
|
// instead inflates head_dim by the grouping factor — eight, on a typical
|
|
// model — and the error lands straight on the cache figure.
|
|
m, err := hfConfig{
|
|
NumHiddenLayers: 48, NumAttentionHeads: 40, NumKeyValueHeads: 8,
|
|
HiddenSize: 5120, MaxPositions: 32768,
|
|
}.toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.HeadDim != 128 {
|
|
t.Errorf("HeadDim = %d, want 5120/40 = 128", m.HeadDim)
|
|
}
|
|
// Which reproduces the figure measured on this cluster.
|
|
if got := m.KVBytesPerToken(BF16); got != 192*1024 {
|
|
t.Errorf("KV/token = %d, want %d", got, 192*1024)
|
|
}
|
|
}
|
|
|
|
func TestAnExplicitHeadDimWins(t *testing.T) {
|
|
// Newer configs state it, and some models do not have hidden_size divisible
|
|
// by the head count — deriving it there is simply wrong.
|
|
m, err := hfConfig{
|
|
NumHiddenLayers: 32, NumAttentionHeads: 32, NumKeyValueHeads: 8,
|
|
HiddenSize: 4096, HeadDim: 256, MaxPositions: 8192,
|
|
}.toModel("org/model")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.HeadDim != 256 {
|
|
t.Errorf("HeadDim = %d; the explicit value should win over 4096/32", m.HeadDim)
|
|
}
|
|
}
|
|
|
|
func TestAConfigWithoutLayersIsRefusedClearly(t *testing.T) {
|
|
// Pointing this at an embedding or vision repository is an easy mistake,
|
|
// and a plan built from zeros would be confidently meaningless.
|
|
_, err := hfConfig{MaxPositions: 512}.toModel("org/not-a-text-model")
|
|
if err == nil {
|
|
t.Fatal("a config with no layers was accepted")
|
|
}
|
|
if !contains(err.Error(), "num_hidden_layers") {
|
|
t.Errorf("the error does not name what is missing: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheModelNameIsTheRepositoryLeaf(t *testing.T) {
|
|
m, _ := hfConfig{NumHiddenLayers: 1, NumAttentionHeads: 1, HiddenSize: 64}.
|
|
toModel("Qwen/Qwen2.5-Coder-14B-Instruct")
|
|
if m.Name != "Qwen2.5-Coder-14B-Instruct" {
|
|
t.Errorf("Name = %q", m.Name)
|
|
}
|
|
}
|