From 4a652fdcc146b1c53267c42d370675531451846b Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Wed, 30 Sep 2026 02:40:00 +0200
Subject: [PATCH] Size a llama.cpp model's memory from its own shape, not its
file size
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The serve form asked for weights plus half again plus two gigabytes. That
ignores the one thing that varies most: the KV cache. At the same 16k pool,
the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two
KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess
overcounted the small models, which on 7 GB nodes means refusing placements
that fit, and undercounted the ones with wide or many KV heads, which means
an engine killed for memory.
The GGUF header already carries the shape. The reader now also takes each
architecture's block_count, head counts, embedding length and key/value
lengths, reading past general.* until the tokenizer — and a header whose
tokenizer comes first and runs past the fetched bytes still returns what it
has rather than failing. The estimate is weights, plus the f16 KV cache for
the context pool (llama.cpp's --ctx-size, which --parallel divides rather
than multiplies), plus a gigabyte and five percent for the engine. It reuses
the header the servability check already fetched, and the form says what it
is made of. Checked against all twelve GGUF models on the live mirror, whose
shapes read back as their published architectures.
Co-Authored-By: Claude Opus 5.5 (1M context)
---
internal/gguf/gguf.go | 126 +++++++++++++++++++++--
internal/gguf/gguf_test.go | 51 +++++++++
internal/gguf/llamacpp.go | 3 +
internal/gguf/remote_test.go | 30 ++++++
internal/web/admin_serve.go | 36 ++++++-
internal/web/templates/admin_mirror.html | 8 +-
6 files changed, 239 insertions(+), 15 deletions(-)
create mode 100644 internal/gguf/remote_test.go
diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go
index 08083e2..906a47c 100644
--- a/internal/gguf/gguf.go
+++ b/internal/gguf/gguf.go
@@ -25,6 +25,59 @@ type Header struct {
Version uint32
Architecture string
Name string
+
+ // The shape the KV cache is sized from, from the architecture's own keys
+ // (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
+ // file does not say, which a caller has to treat as unknown.
+ BlockCount uint64
+ HeadCount uint64
+ HeadCountKV uint64
+ EmbeddingLength uint64
+ // KeyLength and ValueLength are the per-head sizes when the file gives
+ // them; otherwise a head is EmbeddingLength / HeadCount.
+ KeyLength uint64
+ ValueLength uint64
+}
+
+// KVBytesPerToken is what one token of context costs in llama.cpp's default
+// f16 cache, or 0 when the header did not give the shape.
+//
+// Keys and values, per layer, per KV head — the KV heads rather than the
+// attention heads, because with grouped-query attention a model with forty
+// heads may keep only eight, and the difference is a factor of five.
+func (h Header) KVBytesPerToken() int64 {
+ kvHeads := h.HeadCountKV
+ if kvHeads == 0 {
+ kvHeads = h.HeadCount
+ }
+ if h.BlockCount == 0 || kvHeads == 0 {
+ return 0
+ }
+ k, v := h.KeyLength, h.ValueLength
+ if k == 0 && h.HeadCount > 0 {
+ k = h.EmbeddingLength / h.HeadCount
+ }
+ if v == 0 {
+ v = k
+ }
+ if k == 0 {
+ return 0
+ }
+ const f16 = 2
+ return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
+}
+
+// MemoryBytes is what one llama.cpp server needs for this model: the weights,
+// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
+// which --parallel divides among its slots rather than multiplies), and room
+// for compute buffers and the server itself. 0 when the shape is unknown.
+func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
+ perToken := h.KVBytesPerToken()
+ if perToken == 0 || weights <= 0 {
+ return 0
+ }
+ overhead := int64(1<<30) + weights/20
+ return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
@@ -87,16 +140,42 @@ func Read(r io.Reader) (Header, error) {
_, err := io.CopyN(io.Discard, br, n)
return err
}
+ // num holds the last integer value read, for the shape keys.
+ var num uint64
+ readUint := func(size int) error {
+ b := make([]byte, size)
+ if _, err := io.ReadFull(br, b); err != nil {
+ return err
+ }
+ switch size {
+ case 1:
+ num = uint64(b[0])
+ case 2:
+ num = uint64(binary.LittleEndian.Uint16(b))
+ case 4:
+ num = uint64(binary.LittleEndian.Uint32(b))
+ default:
+ num = binary.LittleEndian.Uint64(b)
+ }
+ return nil
+ }
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
+ num = 0
switch t {
- case tUint8, tInt8, tBool:
+ case tUint8, tInt8:
+ return "", readUint(1)
+ case tBool:
return "", skip(1)
case tUint16, tInt16:
- return "", skip(2)
- case tUint32, tInt32, tFloat32:
+ return "", readUint(2)
+ case tUint32, tInt32:
+ return "", readUint(4)
+ case tFloat32:
return "", skip(4)
- case tUint64, tInt64, tFloat64:
+ case tUint64, tInt64:
+ return "", readUint(8)
+ case tFloat64:
return "", skip(8)
case tString:
return str()
@@ -129,18 +208,27 @@ func Read(r io.Reader) (Header, error) {
if err != nil {
return h, err
}
+ // Once the architecture is known, running out of the fetched bytes is
+ // not a failure: what the caller came for is in hand, and the shape keys
+ // are a bonus that a file with its tokenizer first may not reach.
+ partial := func(err error) (Header, error) {
+ if h.Architecture != "" {
+ return h, nil
+ }
+ return h, err
+ }
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
- return h, err
+ return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
- return h, err
+ return partial(err)
}
v, err := value(t)
if err != nil {
- return h, err
+ return partial(err)
}
switch key {
case "general.architecture":
@@ -148,7 +236,25 @@ func Read(r io.Reader) (Header, error) {
case "general.name":
h.Name = v
}
- if h.Architecture != "" && h.Name != "" {
+ if h.Architecture != "" {
+ switch strings.TrimPrefix(key, h.Architecture+".") {
+ case "block_count":
+ h.BlockCount = num
+ case "attention.head_count":
+ h.HeadCount = num
+ case "attention.head_count_kv":
+ h.HeadCountKV = num
+ case "embedding_length":
+ h.EmbeddingLength = num
+ case "attention.key_length":
+ h.KeyLength = num
+ case "attention.value_length":
+ h.ValueLength = num
+ }
+ }
+ // The tokenizer's arrays — megabytes of vocabulary — come after the
+ // architecture's keys, and nothing past them is wanted.
+ if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
@@ -209,5 +315,7 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
- return LlamaCpp(llamaVersion, h.Architecture)
+ v := LlamaCpp(llamaVersion, h.Architecture)
+ v.Header = h
+ return v
}
diff --git a/internal/gguf/gguf_test.go b/internal/gguf/gguf_test.go
index 16f853c..1303cb7 100644
--- a/internal/gguf/gguf_test.go
+++ b/internal/gguf/gguf_test.go
@@ -92,3 +92,54 @@ func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
}
var timeZero = time.Time{}
+
+// shapedHeader is a header with the keys a KV cache is sized from, written
+// with the integer types llama.cpp uses for them.
+func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
+ var b bytes.Buffer
+ w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
+ str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
+ u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
+ b.WriteString("GGUF")
+ w(uint32(3))
+ w(uint64(0))
+ w(uint64(7))
+ str("general.architecture")
+ w(uint32(tString))
+ str(arch)
+ u32(arch+".block_count", layers)
+ u32(arch+".attention.head_count", heads)
+ u32(arch+".attention.head_count_kv", kvHeads)
+ u32(arch+".embedding_length", embd)
+ u32(arch+".context_length", 32768)
+ str("tokenizer.ggml.tokens")
+ w(uint32(tArray))
+ w(uint32(tString))
+ w(uint64(1 << 20))
+ if !truncateTokenizer {
+ for i := 0; i < 1<<20; i++ {
+ str("t")
+ }
+ }
+ return b.Bytes()
+}
+
+func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
+ // Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
+ // 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
+ h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
+ if err != nil {
+ t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
+ }
+ if got := h.KVBytesPerToken(); got != 196608 {
+ t.Errorf("%d bytes a token, expected 196608", got)
+ }
+ weights := int64(9 << 30)
+ want := weights + 16384*196608 + (1<<30 + weights/20)
+ if got := h.MemoryBytes(weights, 16384); got != want {
+ t.Errorf("memory %d, expected %d", got, want)
+ }
+ if (Header{}).MemoryBytes(weights, 16384) != 0 {
+ t.Error("a header with no shape produced a figure")
+ }
+}
diff --git a/internal/gguf/llamacpp.go b/internal/gguf/llamacpp.go
index be31cf3..954ff78 100644
--- a/internal/gguf/llamacpp.go
+++ b/internal/gguf/llamacpp.go
@@ -180,6 +180,9 @@ type Verdict struct {
// checked.
Checked bool
Reason string
+ // Header is what was read, zero when nothing was; its shape sizes the
+ // model's memory.
+ Header Header
}
// LlamaCpp judges an architecture against a llama.cpp build.
diff --git a/internal/gguf/remote_test.go b/internal/gguf/remote_test.go
new file mode 100644
index 0000000..a7f389f
--- /dev/null
+++ b/internal/gguf/remote_test.go
@@ -0,0 +1,30 @@
+package gguf
+
+import (
+ "os"
+ "strings"
+ "testing"
+)
+
+// TestRealHeaders reads real files when LLMGW_GGUF_URLS names them (space
+// separated), and prints the shape and memory estimate for a 16384-token pool.
+// Skipped otherwise: CI has no mirror to read.
+func TestRealHeaders(t *testing.T) {
+ urls := strings.Fields(os.Getenv("LLMGW_GGUF_URLS"))
+ if len(urls) == 0 {
+ t.Skip("LLMGW_GGUF_URLS not set")
+ }
+ for _, u := range urls {
+ h, err := FromURL(t.Context(), nil, u)
+ if err != nil {
+ t.Errorf("%s: %v", u, err)
+ continue
+ }
+ t.Logf("%s: arch=%s layers=%d heads=%d kv_heads=%d embd=%d key_len=%d kv/token=%d B, kv@16k=%.2f GiB",
+ u[strings.LastIndex(u, "/")+1:], h.Architecture, h.BlockCount, h.HeadCount, h.HeadCountKV,
+ h.EmbeddingLength, h.KeyLength, h.KVBytesPerToken(), float64(16384*h.KVBytesPerToken())/(1<<30))
+ if h.KVBytesPerToken() == 0 {
+ t.Errorf("%s: no KV shape read", u)
+ }
+ }
+}
diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go
index 2ff667f..98cfa1e 100644
--- a/internal/web/admin_serve.go
+++ b/internal/web/admin_serve.go
@@ -49,6 +49,11 @@ type serveForm struct {
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
// told what the model needed.
MinMemoryGB int
+ // MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
+ MemoryBasis string
+ // CtxPool is the engine context pool the form starts with, and the one
+ // the memory figure assumes.
+ CtxPool int
// Refusal is set when this version cannot be served as it stands, and says
// why. Offered and then failing at placement is worse than not offered.
@@ -63,6 +68,7 @@ type serveForm struct {
func planServe(st mirror.Status, files []mirror.Status, base string,
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
) serveForm {
+ var header gguf.Header
f := serveForm{
Model: st.Model,
Version: st.Version,
@@ -101,10 +107,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
case "llamacpp":
f.Profile = "cpu"
if check != nil {
- if v := check(f.Source); !v.Servable {
+ v := check(f.Source)
+ if !v.Servable {
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
return f
}
+ header = v.Header
}
default:
f.Profile = "gpu"
@@ -123,7 +131,27 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
"same model runs on the CPU nodes"
}
}
- f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
+ f.CtxPool = defaultCtxPool
+ f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), "weights plus half again plus two"
+ weights := st.BytesTotal
+ if weights <= 0 {
+ weights = st.BytesDone
+ }
+ if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
+ kvHeads := header.HeadCountKV
+ if kvHeads == 0 {
+ kvHeads = header.HeadCount
+ }
+ // From the model's own shape: its KV cache for the pool the form
+ // starts with, which ranges from under half a gigabyte to over five
+ // across the models on the live mirror at the same 16k — a spread the
+ // flat "half again" could not see.
+ f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
+ f.MemoryBasis = fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens (%d layers × %d KV heads) + %.1f GB for the engine",
+ float64(weights)/(1<<30), float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30),
+ defaultCtxPool, header.BlockCount, kvHeads,
+ float64(int64(1<<30)+weights/20)/(1<<30))
+ }
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
f.Warning = strings.TrimSuffix(f.Warning, ".") +
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
@@ -353,3 +381,7 @@ func modeOf(sc catalog.Scaling) string {
return modeStopped
}
}
+
+// defaultCtxPool is the llama.cpp context pool the serve form starts with.
+// --parallel divides it among slots; it does not multiply it.
+const defaultCtxPool = 16384
diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html
index 4f7dce7..64477ca 100644
--- a/internal/web/templates/admin_mirror.html
+++ b/internal/web/templates/admin_mirror.html
@@ -147,11 +147,11 @@ reachable from this machine.