Files
llmgw/internal/gguf/gguf.go
T
Ilker ManapandClaude Opus 5.5 4a652fdcc1 Size a llama.cpp model's memory from its own shape, not its file size
The serve form asked for weights plus half again plus two gigabytes. That
ignores the one thing that varies most: the KV cache. At the same 16k pool,
the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two
KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess
overcounted the small models, which on 7 GB nodes means refusing placements
that fit, and undercounted the ones with wide or many KV heads, which means
an engine killed for memory.

The GGUF header already carries the shape. The reader now also takes each
architecture's block_count, head counts, embedding length and key/value
lengths, reading past general.* until the tokenizer — and a header whose
tokenizer comes first and runs past the fetched bytes still returns what it
has rather than failing. The estimate is weights, plus the f16 KV cache for
the context pool (llama.cpp's --ctx-size, which --parallel divides rather
than multiplies), plus a gigabyte and five percent for the engine. It reuses
the header the servability check already fetched, and the form says what it
is made of. Checked against all twelve GGUF models on the live mirror, whose
shapes read back as their published architectures.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 02:40:00 +02:00

322 lines
8.7 KiB
Go

// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
// The shape the KV cache is sized from, from the architecture's own keys
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
// file does not say, which a caller has to treat as unknown.
BlockCount uint64
HeadCount uint64
HeadCountKV uint64
EmbeddingLength uint64
// KeyLength and ValueLength are the per-head sizes when the file gives
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
// f16 cache, or 0 when the header did not give the shape.
//
// Keys and values, per layer, per KV head — the KV heads rather than the
// attention heads, because with grouped-query attention a model with forty
// heads may keep only eight, and the difference is a factor of five.
func (h Header) KVBytesPerToken() int64 {
kvHeads := h.HeadCountKV
if kvHeads == 0 {
kvHeads = h.HeadCount
}
if h.BlockCount == 0 || kvHeads == 0 {
return 0
}
k, v := h.KeyLength, h.ValueLength
if k == 0 && h.HeadCount > 0 {
k = h.EmbeddingLength / h.HeadCount
}
if v == 0 {
v = k
}
if k == 0 {
return 0
}
const f16 = 2
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
}
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
// which --parallel divides among its slots rather than multiplies), and room
// for compute buffers and the server itself. 0 when the shape is unknown.
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
perToken := h.KVBytesPerToken()
if perToken == 0 || weights <= 0 {
return 0
}
overhead := int64(1<<30) + weights/20
return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
// num holds the last integer value read, for the shape keys.
var num uint64
readUint := func(size int) error {
b := make([]byte, size)
if _, err := io.ReadFull(br, b); err != nil {
return err
}
switch size {
case 1:
num = uint64(b[0])
case 2:
num = uint64(binary.LittleEndian.Uint16(b))
case 4:
num = uint64(binary.LittleEndian.Uint32(b))
default:
num = binary.LittleEndian.Uint64(b)
}
return nil
}
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
num = 0
switch t {
case tUint8, tInt8:
return "", readUint(1)
case tBool:
return "", skip(1)
case tUint16, tInt16:
return "", readUint(2)
case tUint32, tInt32:
return "", readUint(4)
case tFloat32:
return "", skip(4)
case tUint64, tInt64:
return "", readUint(8)
case tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
// Once the architecture is known, running out of the fetched bytes is
// not a failure: what the caller came for is in hand, and the shape keys
// are a bonus that a file with its tokenizer first may not reach.
partial := func(err error) (Header, error) {
if h.Architecture != "" {
return h, nil
}
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return partial(err)
}
v, err := value(t)
if err != nil {
return partial(err)
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
}
if h.Architecture != "" {
switch strings.TrimPrefix(key, h.Architecture+".") {
case "block_count":
h.BlockCount = num
case "attention.head_count":
h.HeadCount = num
case "attention.head_count_kv":
h.HeadCountKV = num
case "embedding_length":
h.EmbeddingLength = num
case "attention.key_length":
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
// architecture's keys, and nothing past them is wanted.
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
v := LlamaCpp(llamaVersion, h.Architecture)
v.Header = h
return v
}