Size a llama.cpp model's memory from its own shape, not its file size
The serve form asked for weights plus half again plus two gigabytes. That ignores the one thing that varies most: the KV cache. At the same 16k pool, the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess overcounted the small models, which on 7 GB nodes means refusing placements that fit, and undercounted the ones with wide or many KV heads, which means an engine killed for memory. The GGUF header already carries the shape. The reader now also takes each architecture's block_count, head counts, embedding length and key/value lengths, reading past general.* until the tokenizer — and a header whose tokenizer comes first and runs past the fetched bytes still returns what it has rather than failing. The estimate is weights, plus the f16 KV cache for the context pool (llama.cpp's --ctx-size, which --parallel divides rather than multiplies), plus a gigabyte and five percent for the engine. It reuses the header the servability check already fetched, and the form says what it is made of. Checked against all twelve GGUF models on the live mirror, whose shapes read back as their published architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f8572fb5ee
commit
4a652fdcc1
6 files changed
+239
-15
No files matched your search
+117
-9
@@ -25,6 +25,59 @@ type Header struct {
|
||||
Version uint32
|
||||
Architecture string
|
||||
Name string
|
||||
|
||||
// The shape the KV cache is sized from, from the architecture's own keys
|
||||
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
|
||||
// file does not say, which a caller has to treat as unknown.
|
||||
BlockCount uint64
|
||||
HeadCount uint64
|
||||
HeadCountKV uint64
|
||||
EmbeddingLength uint64
|
||||
// KeyLength and ValueLength are the per-head sizes when the file gives
|
||||
// them; otherwise a head is EmbeddingLength / HeadCount.
|
||||
KeyLength uint64
|
||||
ValueLength uint64
|
||||
}
|
||||
|
||||
// KVBytesPerToken is what one token of context costs in llama.cpp's default
|
||||
// f16 cache, or 0 when the header did not give the shape.
|
||||
//
|
||||
// Keys and values, per layer, per KV head — the KV heads rather than the
|
||||
// attention heads, because with grouped-query attention a model with forty
|
||||
// heads may keep only eight, and the difference is a factor of five.
|
||||
func (h Header) KVBytesPerToken() int64 {
|
||||
kvHeads := h.HeadCountKV
|
||||
if kvHeads == 0 {
|
||||
kvHeads = h.HeadCount
|
||||
}
|
||||
if h.BlockCount == 0 || kvHeads == 0 {
|
||||
return 0
|
||||
}
|
||||
k, v := h.KeyLength, h.ValueLength
|
||||
if k == 0 && h.HeadCount > 0 {
|
||||
k = h.EmbeddingLength / h.HeadCount
|
||||
}
|
||||
if v == 0 {
|
||||
v = k
|
||||
}
|
||||
if k == 0 {
|
||||
return 0
|
||||
}
|
||||
const f16 = 2
|
||||
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
|
||||
}
|
||||
|
||||
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
|
||||
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
|
||||
// which --parallel divides among its slots rather than multiplies), and room
|
||||
// for compute buffers and the server itself. 0 when the shape is unknown.
|
||||
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
|
||||
perToken := h.KVBytesPerToken()
|
||||
if perToken == 0 || weights <= 0 {
|
||||
return 0
|
||||
}
|
||||
overhead := int64(1<<30) + weights/20
|
||||
return weights + int64(ctxPool)*perToken + overhead
|
||||
}
|
||||
|
||||
// ErrNotGGUF means the bytes are not a GGUF file.
|
||||
@@ -87,16 +140,42 @@ func Read(r io.Reader) (Header, error) {
|
||||
_, err := io.CopyN(io.Discard, br, n)
|
||||
return err
|
||||
}
|
||||
// num holds the last integer value read, for the shape keys.
|
||||
var num uint64
|
||||
readUint := func(size int) error {
|
||||
b := make([]byte, size)
|
||||
if _, err := io.ReadFull(br, b); err != nil {
|
||||
return err
|
||||
}
|
||||
switch size {
|
||||
case 1:
|
||||
num = uint64(b[0])
|
||||
case 2:
|
||||
num = uint64(binary.LittleEndian.Uint16(b))
|
||||
case 4:
|
||||
num = uint64(binary.LittleEndian.Uint32(b))
|
||||
default:
|
||||
num = binary.LittleEndian.Uint64(b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var value func(t uint32) (string, error)
|
||||
value = func(t uint32) (string, error) {
|
||||
num = 0
|
||||
switch t {
|
||||
case tUint8, tInt8, tBool:
|
||||
case tUint8, tInt8:
|
||||
return "", readUint(1)
|
||||
case tBool:
|
||||
return "", skip(1)
|
||||
case tUint16, tInt16:
|
||||
return "", skip(2)
|
||||
case tUint32, tInt32, tFloat32:
|
||||
return "", readUint(2)
|
||||
case tUint32, tInt32:
|
||||
return "", readUint(4)
|
||||
case tFloat32:
|
||||
return "", skip(4)
|
||||
case tUint64, tInt64, tFloat64:
|
||||
case tUint64, tInt64:
|
||||
return "", readUint(8)
|
||||
case tFloat64:
|
||||
return "", skip(8)
|
||||
case tString:
|
||||
return str()
|
||||
@@ -129,18 +208,27 @@ func Read(r io.Reader) (Header, error) {
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
// Once the architecture is known, running out of the fetched bytes is
|
||||
// not a failure: what the caller came for is in hand, and the shape keys
|
||||
// are a bonus that a file with its tokenizer first may not reach.
|
||||
partial := func(err error) (Header, error) {
|
||||
if h.Architecture != "" {
|
||||
return h, nil
|
||||
}
|
||||
return h, err
|
||||
}
|
||||
for i := uint64(0); i < kvs; i++ {
|
||||
key, err := str()
|
||||
if err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
var t uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
v, err := value(t)
|
||||
if err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
switch key {
|
||||
case "general.architecture":
|
||||
@@ -148,7 +236,25 @@ func Read(r io.Reader) (Header, error) {
|
||||
case "general.name":
|
||||
h.Name = v
|
||||
}
|
||||
if h.Architecture != "" && h.Name != "" {
|
||||
if h.Architecture != "" {
|
||||
switch strings.TrimPrefix(key, h.Architecture+".") {
|
||||
case "block_count":
|
||||
h.BlockCount = num
|
||||
case "attention.head_count":
|
||||
h.HeadCount = num
|
||||
case "attention.head_count_kv":
|
||||
h.HeadCountKV = num
|
||||
case "embedding_length":
|
||||
h.EmbeddingLength = num
|
||||
case "attention.key_length":
|
||||
h.KeyLength = num
|
||||
case "attention.value_length":
|
||||
h.ValueLength = num
|
||||
}
|
||||
}
|
||||
// The tokenizer's arrays — megabytes of vocabulary — come after the
|
||||
// architecture's keys, and nothing past them is wanted.
|
||||
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -209,5 +315,7 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
||||
if err != nil {
|
||||
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
||||
}
|
||||
return LlamaCpp(llamaVersion, h.Architecture)
|
||||
v := LlamaCpp(llamaVersion, h.Architecture)
|
||||
v.Header = h
|
||||
return v
|
||||
}
|
||||
@@ -92,3 +92,54 @@ func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
var timeZero = time.Time{}
|
||||
|
||||
// shapedHeader is a header with the keys a KV cache is sized from, written
|
||||
// with the integer types llama.cpp uses for them.
|
||||
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(7))
|
||||
str("general.architecture")
|
||||
w(uint32(tString))
|
||||
str(arch)
|
||||
u32(arch+".block_count", layers)
|
||||
u32(arch+".attention.head_count", heads)
|
||||
u32(arch+".attention.head_count_kv", kvHeads)
|
||||
u32(arch+".embedding_length", embd)
|
||||
u32(arch+".context_length", 32768)
|
||||
str("tokenizer.ggml.tokens")
|
||||
w(uint32(tArray))
|
||||
w(uint32(tString))
|
||||
w(uint64(1 << 20))
|
||||
if !truncateTokenizer {
|
||||
for i := 0; i < 1<<20; i++ {
|
||||
str("t")
|
||||
}
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
|
||||
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
|
||||
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
|
||||
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
|
||||
if err != nil {
|
||||
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
|
||||
}
|
||||
if got := h.KVBytesPerToken(); got != 196608 {
|
||||
t.Errorf("%d bytes a token, expected 196608", got)
|
||||
}
|
||||
weights := int64(9 << 30)
|
||||
want := weights + 16384*196608 + (1<<30 + weights/20)
|
||||
if got := h.MemoryBytes(weights, 16384); got != want {
|
||||
t.Errorf("memory %d, expected %d", got, want)
|
||||
}
|
||||
if (Header{}).MemoryBytes(weights, 16384) != 0 {
|
||||
t.Error("a header with no shape produced a figure")
|
||||
}
|
||||
}
|
||||
@@ -180,6 +180,9 @@ type Verdict struct {
|
||||
// checked.
|
||||
Checked bool
|
||||
Reason string
|
||||
// Header is what was read, zero when nothing was; its shape sizes the
|
||||
// model's memory.
|
||||
Header Header
|
||||
}
|
||||
|
||||
// LlamaCpp judges an architecture against a llama.cpp build.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRealHeaders reads real files when LLMGW_GGUF_URLS names them (space
|
||||
// separated), and prints the shape and memory estimate for a 16384-token pool.
|
||||
// Skipped otherwise: CI has no mirror to read.
|
||||
func TestRealHeaders(t *testing.T) {
|
||||
urls := strings.Fields(os.Getenv("LLMGW_GGUF_URLS"))
|
||||
if len(urls) == 0 {
|
||||
t.Skip("LLMGW_GGUF_URLS not set")
|
||||
}
|
||||
for _, u := range urls {
|
||||
h, err := FromURL(t.Context(), nil, u)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", u, err)
|
||||
continue
|
||||
}
|
||||
t.Logf("%s: arch=%s layers=%d heads=%d kv_heads=%d embd=%d key_len=%d kv/token=%d B, kv@16k=%.2f GiB",
|
||||
u[strings.LastIndex(u, "/")+1:], h.Architecture, h.BlockCount, h.HeadCount, h.HeadCountKV,
|
||||
h.EmbeddingLength, h.KeyLength, h.KVBytesPerToken(), float64(16384*h.KVBytesPerToken())/(1<<30))
|
||||
if h.KVBytesPerToken() == 0 {
|
||||
t.Errorf("%s: no KV shape read", u)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,11 @@ type serveForm struct {
|
||||
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
|
||||
// told what the model needed.
|
||||
MinMemoryGB int
|
||||
// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
|
||||
MemoryBasis string
|
||||
// CtxPool is the engine context pool the form starts with, and the one
|
||||
// the memory figure assumes.
|
||||
CtxPool int
|
||||
|
||||
// Refusal is set when this version cannot be served as it stands, and says
|
||||
// why. Offered and then failing at placement is worse than not offered.
|
||||
@@ -63,6 +68,7 @@ type serveForm struct {
|
||||
func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
|
||||
) serveForm {
|
||||
var header gguf.Header
|
||||
f := serveForm{
|
||||
Model: st.Model,
|
||||
Version: st.Version,
|
||||
@@ -101,10 +107,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
case "llamacpp":
|
||||
f.Profile = "cpu"
|
||||
if check != nil {
|
||||
if v := check(f.Source); !v.Servable {
|
||||
v := check(f.Source)
|
||||
if !v.Servable {
|
||||
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
|
||||
return f
|
||||
}
|
||||
header = v.Header
|
||||
}
|
||||
default:
|
||||
f.Profile = "gpu"
|
||||
@@ -123,7 +131,27 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
"same model runs on the CPU nodes"
|
||||
}
|
||||
}
|
||||
f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
|
||||
f.CtxPool = defaultCtxPool
|
||||
f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), "weights plus half again plus two"
|
||||
weights := st.BytesTotal
|
||||
if weights <= 0 {
|
||||
weights = st.BytesDone
|
||||
}
|
||||
if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
|
||||
kvHeads := header.HeadCountKV
|
||||
if kvHeads == 0 {
|
||||
kvHeads = header.HeadCount
|
||||
}
|
||||
// From the model's own shape: its KV cache for the pool the form
|
||||
// starts with, which ranges from under half a gigabyte to over five
|
||||
// across the models on the live mirror at the same 16k — a spread the
|
||||
// flat "half again" could not see.
|
||||
f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
|
||||
f.MemoryBasis = fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens (%d layers × %d KV heads) + %.1f GB for the engine",
|
||||
float64(weights)/(1<<30), float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30),
|
||||
defaultCtxPool, header.BlockCount, kvHeads,
|
||||
float64(int64(1<<30)+weights/20)/(1<<30))
|
||||
}
|
||||
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
|
||||
f.Warning = strings.TrimSuffix(f.Warning, ".") +
|
||||
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
|
||||
@@ -353,3 +381,7 @@ func modeOf(sc catalog.Scaling) string {
|
||||
return modeStopped
|
||||
}
|
||||
}
|
||||
|
||||
// defaultCtxPool is the llama.cpp context pool the serve form starts with.
|
||||
// --parallel divides it among slots; it does not multiply it.
|
||||
const defaultCtxPool = 16384
|
||||
@@ -147,11 +147,11 @@ reachable from this machine.</p>
|
||||
<label>Output limit <span class="opt">optional</span>
|
||||
<input type="text" name="max_output" value="1024"></label>
|
||||
<label>Engine context pool <span class="opt">divided across slots</span>
|
||||
<input type="text" name="ctx_size" value="16384"></label>
|
||||
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
|
||||
</div>
|
||||
<div class="field-row">
|
||||
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
|
||||
<label>Needs RAM (GB) <span class="opt">estimated from the weights</span>
|
||||
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
|
||||
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}"></label>
|
||||
<label>Only nodes labelled <span class="opt">key / value, optional</span>
|
||||
<input type="text" name="selector_key" placeholder="role" size="8">
|
||||
@@ -162,8 +162,8 @@ reachable from this machine.</p>
|
||||
<p class="note">The context pool is what the engine is given; llama.cpp divides
|
||||
it across its parallel slots, so the number one request may use is the smaller
|
||||
one above.</p>
|
||||
<p class="note">The memory figure is the weights plus half again for the KV pool
|
||||
plus two for the engine — an estimate, and editable. It is filled in rather than
|
||||
<p class="note">Memory: {{.MemoryBasis}}. Change the context pool and this changes
|
||||
with it, by the KV cache's share — an estimate, and editable. It is filled in rather than
|
||||
left blank because a CPU profile has no card to be short of: with no figure,
|
||||
every ready node is a candidate and the first one picked is whichever the
|
||||
scheduler reached, not whichever can hold the model. A node label does the same
|
||||
|
||||
Reference in new issue
Block a user