Size a llama.cpp model's memory from its own shape, not its file size

The serve form asked for weights plus half again plus two gigabytes. That
ignores the one thing that varies most: the KV cache. At the same 16k pool,
the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two
KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess
overcounted the small models, which on 7 GB nodes means refusing placements
that fit, and undercounted the ones with wide or many KV heads, which means
an engine killed for memory.

The GGUF header already carries the shape. The reader now also takes each
architecture's block_count, head counts, embedding length and key/value
lengths, reading past general.* until the tokenizer — and a header whose
tokenizer comes first and runs past the fetched bytes still returns what it
has rather than failing. The estimate is weights, plus the f16 KV cache for
the context pool (llama.cpp's --ctx-size, which --parallel divides rather
than multiplies), plus a gigabyte and five percent for the engine. It reuses
the header the servability check already fetched, and the form says what it
is made of. Checked against all twelve GGUF models on the live mirror, whose
shapes read back as their published architectures.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 02:40:00 +02:00
1 parent f8572fb5ee
commit 4a652fdcc1
6 files changed
+239 -15

No files matched your search

+117 -9
View File
@@ -25,6 +25,59 @@ type Header struct {
Version uint32
Architecture string
Name string
// The shape the KV cache is sized from, from the architecture's own keys
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
// file does not say, which a caller has to treat as unknown.
BlockCount uint64
HeadCount uint64
HeadCountKV uint64
EmbeddingLength uint64
// KeyLength and ValueLength are the per-head sizes when the file gives
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
// f16 cache, or 0 when the header did not give the shape.
//
// Keys and values, per layer, per KV head — the KV heads rather than the
// attention heads, because with grouped-query attention a model with forty
// heads may keep only eight, and the difference is a factor of five.
func (h Header) KVBytesPerToken() int64 {
kvHeads := h.HeadCountKV
if kvHeads == 0 {
kvHeads = h.HeadCount
}
if h.BlockCount == 0 || kvHeads == 0 {
return 0
}
k, v := h.KeyLength, h.ValueLength
if k == 0 && h.HeadCount > 0 {
k = h.EmbeddingLength / h.HeadCount
}
if v == 0 {
v = k
}
if k == 0 {
return 0
}
const f16 = 2
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
}
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
// which --parallel divides among its slots rather than multiplies), and room
// for compute buffers and the server itself. 0 when the shape is unknown.
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
perToken := h.KVBytesPerToken()
if perToken == 0 || weights <= 0 {
return 0
}
overhead := int64(1<<30) + weights/20
return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
@@ -87,16 +140,42 @@ func Read(r io.Reader) (Header, error) {
_, err := io.CopyN(io.Discard, br, n)
return err
}
// num holds the last integer value read, for the shape keys.
var num uint64
readUint := func(size int) error {
b := make([]byte, size)
if _, err := io.ReadFull(br, b); err != nil {
return err
}
switch size {
case 1:
num = uint64(b[0])
case 2:
num = uint64(binary.LittleEndian.Uint16(b))
case 4:
num = uint64(binary.LittleEndian.Uint32(b))
default:
num = binary.LittleEndian.Uint64(b)
}
return nil
}
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
num = 0
switch t {
case tUint8, tInt8, tBool:
case tUint8, tInt8:
return "", readUint(1)
case tBool:
return "", skip(1)
case tUint16, tInt16:
return "", skip(2)
case tUint32, tInt32, tFloat32:
return "", readUint(2)
case tUint32, tInt32:
return "", readUint(4)
case tFloat32:
return "", skip(4)
case tUint64, tInt64, tFloat64:
case tUint64, tInt64:
return "", readUint(8)
case tFloat64:
return "", skip(8)
case tString:
return str()
@@ -129,18 +208,27 @@ func Read(r io.Reader) (Header, error) {
if err != nil {
return h, err
}
// Once the architecture is known, running out of the fetched bytes is
// not a failure: what the caller came for is in hand, and the shape keys
// are a bonus that a file with its tokenizer first may not reach.
partial := func(err error) (Header, error) {
if h.Architecture != "" {
return h, nil
}
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return h, err
return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return h, err
return partial(err)
}
v, err := value(t)
if err != nil {
return h, err
return partial(err)
}
switch key {
case "general.architecture":
@@ -148,7 +236,25 @@ func Read(r io.Reader) (Header, error) {
case "general.name":
h.Name = v
}
if h.Architecture != "" && h.Name != "" {
if h.Architecture != "" {
switch strings.TrimPrefix(key, h.Architecture+".") {
case "block_count":
h.BlockCount = num
case "attention.head_count":
h.HeadCount = num
case "attention.head_count_kv":
h.HeadCountKV = num
case "embedding_length":
h.EmbeddingLength = num
case "attention.key_length":
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
// architecture's keys, and nothing past them is wanted.
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
@@ -209,5 +315,7 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
return LlamaCpp(llamaVersion, h.Architecture)
v := LlamaCpp(llamaVersion, h.Architecture)
v.Header = h
return v
}
+51
View File
@@ -92,3 +92,54 @@ func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
}
var timeZero = time.Time{}
// shapedHeader is a header with the keys a KV cache is sized from, written
// with the integer types llama.cpp uses for them.
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(7))
str("general.architecture")
w(uint32(tString))
str(arch)
u32(arch+".block_count", layers)
u32(arch+".attention.head_count", heads)
u32(arch+".attention.head_count_kv", kvHeads)
u32(arch+".embedding_length", embd)
u32(arch+".context_length", 32768)
str("tokenizer.ggml.tokens")
w(uint32(tArray))
w(uint32(tString))
w(uint64(1 << 20))
if !truncateTokenizer {
for i := 0; i < 1<<20; i++ {
str("t")
}
}
return b.Bytes()
}
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
if err != nil {
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
}
if got := h.KVBytesPerToken(); got != 196608 {
t.Errorf("%d bytes a token, expected 196608", got)
}
weights := int64(9 << 30)
want := weights + 16384*196608 + (1<<30 + weights/20)
if got := h.MemoryBytes(weights, 16384); got != want {
t.Errorf("memory %d, expected %d", got, want)
}
if (Header{}).MemoryBytes(weights, 16384) != 0 {
t.Error("a header with no shape produced a figure")
}
}
+3
View File
@@ -180,6 +180,9 @@ type Verdict struct {
// checked.
Checked bool
Reason string
// Header is what was read, zero when nothing was; its shape sizes the
// model's memory.
Header Header
}
// LlamaCpp judges an architecture against a llama.cpp build.
+30
View File
@@ -0,0 +1,30 @@
package gguf
import (
"os"
"strings"
"testing"
)
// TestRealHeaders reads real files when LLMGW_GGUF_URLS names them (space
// separated), and prints the shape and memory estimate for a 16384-token pool.
// Skipped otherwise: CI has no mirror to read.
func TestRealHeaders(t *testing.T) {
urls := strings.Fields(os.Getenv("LLMGW_GGUF_URLS"))
if len(urls) == 0 {
t.Skip("LLMGW_GGUF_URLS not set")
}
for _, u := range urls {
h, err := FromURL(t.Context(), nil, u)
if err != nil {
t.Errorf("%s: %v", u, err)
continue
}
t.Logf("%s: arch=%s layers=%d heads=%d kv_heads=%d embd=%d key_len=%d kv/token=%d B, kv@16k=%.2f GiB",
u[strings.LastIndex(u, "/")+1:], h.Architecture, h.BlockCount, h.HeadCount, h.HeadCountKV,
h.EmbeddingLength, h.KeyLength, h.KVBytesPerToken(), float64(16384*h.KVBytesPerToken())/(1<<30))
if h.KVBytesPerToken() == 0 {
t.Errorf("%s: no KV shape read", u)
}
}
}
+34 -2
View File
@@ -49,6 +49,11 @@ type serveForm struct {
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
// told what the model needed.
MinMemoryGB int
// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
MemoryBasis string
// CtxPool is the engine context pool the form starts with, and the one
// the memory figure assumes.
CtxPool int
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
@@ -63,6 +68,7 @@ type serveForm struct {
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
) serveForm {
var header gguf.Header
f := serveForm{
Model: st.Model,
Version: st.Version,
@@ -101,10 +107,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
case "llamacpp":
f.Profile = "cpu"
if check != nil {
if v := check(f.Source); !v.Servable {
v := check(f.Source)
if !v.Servable {
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
return f
}
header = v.Header
}
default:
f.Profile = "gpu"
@@ -123,7 +131,27 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
"same model runs on the CPU nodes"
}
}
f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
f.CtxPool = defaultCtxPool
f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), "weights plus half again plus two"
weights := st.BytesTotal
if weights <= 0 {
weights = st.BytesDone
}
if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
kvHeads := header.HeadCountKV
if kvHeads == 0 {
kvHeads = header.HeadCount
}
// From the model's own shape: its KV cache for the pool the form
// starts with, which ranges from under half a gigabyte to over five
// across the models on the live mirror at the same 16k — a spread the
// flat "half again" could not see.
f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
f.MemoryBasis = fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens (%d layers × %d KV heads) + %.1f GB for the engine",
float64(weights)/(1<<30), float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30),
defaultCtxPool, header.BlockCount, kvHeads,
float64(int64(1<<30)+weights/20)/(1<<30))
}
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
f.Warning = strings.TrimSuffix(f.Warning, ".") +
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
@@ -353,3 +381,7 @@ func modeOf(sc catalog.Scaling) string {
return modeStopped
}
}
// defaultCtxPool is the llama.cpp context pool the serve form starts with.
// --parallel divides it among slots; it does not multiply it.
const defaultCtxPool = 16384
+4 -4
View File
@@ -147,11 +147,11 @@ reachable from this machine.</p>
<label>Output limit <span class="opt">optional</span>
<input type="text" name="max_output" value="1024"></label>
<label>Engine context pool <span class="opt">divided across slots</span>
<input type="text" name="ctx_size" value="16384"></label>
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
</div>
<div class="field-row">
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
<label>Needs RAM (GB) <span class="opt">estimated from the weights</span>
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}"></label>
<label>Only nodes labelled <span class="opt">key / value, optional</span>
<input type="text" name="selector_key" placeholder="role" size="8">
@@ -162,8 +162,8 @@ reachable from this machine.</p>
<p class="note">The context pool is what the engine is given; llama.cpp divides
it across its parallel slots, so the number one request may use is the smaller
one above.</p>
<p class="note">The memory figure is the weights plus half again for the KV pool
plus two for the engine — an estimate, and editable. It is filled in rather than
<p class="note">Memory: {{.MemoryBasis}}. Change the context pool and this changes
with it, by the KV cache's share — an estimate, and editable. It is filled in rather than
left blank because a CPU profile has no card to be short of: with no figure,
every ready node is a candidate and the first one picked is whichever the
scheduler reached, not whichever can hold the model. A node label does the same