Files
llmgw/internal/gguf/gguf.go
T
Ilker ManapandClaude Opus 5.5 6403e61f4b
CI / test (push) Successful in 9m31s
How fast a model answers on the CPU, before it is served
Each agent measures one core's memory rate beside the whole machine's; the
estimate takes a replica's threads at about one core's rate each, up to the
machine's, calibrated by llama-bench on the PoC. The serve form shows it
beside a card's, warns when slow, and asks for the CPU on purpose when a
reply would take minutes; ctl model plan, the console and serve jobs say
it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 08:52:15 +01:00

443 lines
13 KiB
Go

// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
// Type is general.type: "model", or "adapter" for a LoRA adapter, which
// is applied to a model rather than run on its own. Empty in files made
// before the key existed, which are models.
Type string
// The shape the KV cache is sized from, from the architecture's own keys
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
// file does not say, which a caller has to treat as unknown.
BlockCount uint64
HeadCount uint64
HeadCountKV uint64
EmbeddingLength uint64
// KeyLength and ValueLength are the per-head sizes when the file gives
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
// ContextLength is the context the model was trained for
// (<arch>.context_length); 0 when the file does not say.
ContextLength uint64
// Embedding is whether the model pools its tokens into one vector
// (<arch>.pooling_type set and not none): an embedding model, which
// llama.cpp serves with --embedding and does not chat.
Embedding bool
// ExpertCount and ExpertUsedCount are a model of experts' experts and
// how many each token uses (<arch>.expert_count, expert_used_count);
// zero for a dense model.
ExpertCount, ExpertUsedCount uint64
}
// ActiveShare is the part of the weights a token reads: for a model of
// experts its experts' share plus a tenth for what every token reads
// whatever the experts — attention, embeddings, routing, shared experts.
// Qwen3-Coder-30B-A3B, eight of 128, read 0.16 of its weights a token on
// cuda01's card, which is 0.0625 and the tenth. 1 for a dense model.
func (h Header) ActiveShare() float64 {
if h.ExpertCount == 0 || h.ExpertUsedCount == 0 || h.ExpertUsedCount >= h.ExpertCount {
return 1
}
return min(1, float64(h.ExpertUsedCount)/float64(h.ExpertCount)+0.1)
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
// f16 cache, or 0 when the header did not give the shape.
//
// Keys and values, per layer, per KV head — the KV heads rather than the
// attention heads, because with grouped-query attention a model with forty
// heads may keep only eight, and the difference is a factor of five.
func (h Header) KVBytesPerToken() int64 {
kvHeads := h.HeadCountKV
if kvHeads == 0 {
kvHeads = h.HeadCount
}
if h.BlockCount == 0 || kvHeads == 0 {
return 0
}
k, v := h.KeyLength, h.ValueLength
if k == 0 && h.HeadCount > 0 {
k = h.EmbeddingLength / h.HeadCount
}
if v == 0 {
v = k
}
if k == 0 {
return 0
}
const f16 = 2
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
}
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
// which --parallel divides among its slots rather than multiplies), and room
// for compute buffers and the server itself. 0 when the shape is unknown.
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
perToken := h.KVBytesPerToken()
if perToken == 0 || weights <= 0 {
return 0
}
overhead := int64(1<<30) + weights/20
return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
// num holds the last integer value read, for the shape keys.
var num uint64
readUint := func(size int) error {
b := make([]byte, size)
if _, err := io.ReadFull(br, b); err != nil {
return err
}
switch size {
case 1:
num = uint64(b[0])
case 2:
num = uint64(binary.LittleEndian.Uint16(b))
case 4:
num = uint64(binary.LittleEndian.Uint32(b))
default:
num = binary.LittleEndian.Uint64(b)
}
return nil
}
// depth is how many arrays the value being read is inside. Real files
// nest none; a crafted one nesting thousands deep made a recursion as
// deep, a few bytes per level.
depth := 0
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
num = 0
switch t {
case tUint8, tInt8:
return "", readUint(1)
case tBool:
return "", skip(1)
case tUint16, tInt16:
return "", readUint(2)
case tUint32, tInt32:
return "", readUint(4)
case tFloat32:
return "", skip(4)
case tUint64, tInt64:
return "", readUint(8)
case tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
if depth >= maxArrayDepth {
return "", fmt.Errorf("arrays nested more than %d deep in a GGUF header", maxArrayDepth)
}
depth++
defer func() { depth-- }()
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
// Once the architecture is known, running out of the fetched bytes is
// not a failure: what the caller came for is in hand, and the shape keys
// are a bonus that a file with its tokenizer first may not reach.
partial := func(err error) (Header, error) {
if h.Architecture != "" {
return h, nil
}
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return partial(err)
}
v, err := value(t)
if err != nil {
return partial(err)
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
case "general.type":
h.Type = v
}
if h.Architecture != "" {
switch strings.TrimPrefix(key, h.Architecture+".") {
case "block_count":
h.BlockCount = num
case "attention.head_count":
h.HeadCount = num
case "attention.head_count_kv":
h.HeadCountKV = num
case "embedding_length":
h.EmbeddingLength = num
case "attention.key_length":
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
case "pooling_type":
h.Embedding = num > 0
case "context_length":
h.ContextLength = num
case "expert_count":
h.ExpertCount = num
case "expert_used_count":
h.ExpertUsedCount = num
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
// architecture's keys, and nothing past them is wanted.
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// maxArrayDepth is how deep arrays may nest in a header.
const maxArrayDepth = 4
// IsAdapter reports whether the file is a LoRA adapter rather than a model.
func (h Header) IsAdapter() bool { return h.Type == "adapter" }
// adapterProbe is how much of a file is read to tell an adapter from a model:
// general.type comes before the tokenizer, which is what makes a model's
// header megabytes long.
const adapterProbe = 256 << 10
// KindFromURL reads just enough of a GGUF file to say what it is — its type
// and architecture — over HTTP. A header cut short by the probe is fine: the
// keys asked for come first.
func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
h, err := Read(io.LimitReader(resp.Body, adapterProbe))
if h.Architecture != "" {
return h, nil // what was asked for was read, whatever came after
}
return h, err
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
v := LlamaCpp(llamaVersion, h.Architecture)
v.Header = h
return v
}
// CardUsable is the share of a card's memory a model may take: the rest is
// CUDA's own and the engine's compute buffers.
const CardUsable = 0.9
// PoolThatFits is the largest context pool, in tokens, whose KV cache fits
// on gpus cards of cardGB beside weights bytes of weights and a gigabyte for
// the engine: -1 when the weights alone do not, 0 when the header or the
// card does not say enough to tell.
func (h Header) PoolThatFits(weights int64, cardGB, gpus int) int {
perTok := h.KVBytesPerToken()
if cardGB <= 0 || perTok <= 0 || weights <= 0 {
return 0
}
budget := float64(cardGB*max(gpus, 1))*CardUsable*(1<<30) - float64(weights) - (1 << 30)
if budget <= 0 {
return -1
}
return int(budget / float64(perTok))
}
// ClassMemGB is a card's memory as its class names it: 6000-24g is 24, 0
// for a class that does not say.
func ClassMemGB(class string) int {
i := strings.LastIndex(class, "-")
if i < 0 || !strings.HasSuffix(class, "g") {
return 0
}
n := 0
for _, c := range strings.TrimSuffix(class[i+1:], "g") {
if c < '0' || c > '9' {
return 0
}
n = n*10 + int(c-'0')
}
return n
}