The serve form asked for weights plus half again plus two gigabytes. That ignores the one thing that varies most: the KV cache. At the same 16k pool, the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess overcounted the small models, which on 7 GB nodes means refusing placements that fit, and undercounted the ones with wide or many KV heads, which means an engine killed for memory. The GGUF header already carries the shape. The reader now also takes each architecture's block_count, head counts, embedding length and key/value lengths, reading past general.* until the tokenizer — and a header whose tokenizer comes first and runs past the fetched bytes still returns what it has rather than failing. The estimate is weights, plus the f16 KV cache for the context pool (llama.cpp's --ctx-size, which --parallel divides rather than multiplies), plus a gigabyte and five percent for the engine. It reuses the header the servability check already fetched, and the form says what it is made of. Checked against all twelve GGUF models on the live mirror, whose shapes read back as their published architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
322 lines
8.7 KiB
Go
322 lines
8.7 KiB
Go
// Package gguf reads what a GGUF file says about itself, from its header.
|
|
//
|
|
// Enough to ask one question before anything is downloaded or served: can the
|
|
// engine load this? A file downloaded, catalogued and placed, and then refused
|
|
// by llama.cpp with "unknown model architecture", costs a download and a
|
|
// replica failing on every node it is tried on — and the answer was in the
|
|
// first few kilobytes all along.
|
|
package gguf
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Header is the part of a GGUF header this package reads.
|
|
type Header struct {
|
|
Version uint32
|
|
Architecture string
|
|
Name string
|
|
|
|
// The shape the KV cache is sized from, from the architecture's own keys
|
|
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
|
|
// file does not say, which a caller has to treat as unknown.
|
|
BlockCount uint64
|
|
HeadCount uint64
|
|
HeadCountKV uint64
|
|
EmbeddingLength uint64
|
|
// KeyLength and ValueLength are the per-head sizes when the file gives
|
|
// them; otherwise a head is EmbeddingLength / HeadCount.
|
|
KeyLength uint64
|
|
ValueLength uint64
|
|
}
|
|
|
|
// KVBytesPerToken is what one token of context costs in llama.cpp's default
|
|
// f16 cache, or 0 when the header did not give the shape.
|
|
//
|
|
// Keys and values, per layer, per KV head — the KV heads rather than the
|
|
// attention heads, because with grouped-query attention a model with forty
|
|
// heads may keep only eight, and the difference is a factor of five.
|
|
func (h Header) KVBytesPerToken() int64 {
|
|
kvHeads := h.HeadCountKV
|
|
if kvHeads == 0 {
|
|
kvHeads = h.HeadCount
|
|
}
|
|
if h.BlockCount == 0 || kvHeads == 0 {
|
|
return 0
|
|
}
|
|
k, v := h.KeyLength, h.ValueLength
|
|
if k == 0 && h.HeadCount > 0 {
|
|
k = h.EmbeddingLength / h.HeadCount
|
|
}
|
|
if v == 0 {
|
|
v = k
|
|
}
|
|
if k == 0 {
|
|
return 0
|
|
}
|
|
const f16 = 2
|
|
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
|
|
}
|
|
|
|
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
|
|
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
|
|
// which --parallel divides among its slots rather than multiplies), and room
|
|
// for compute buffers and the server itself. 0 when the shape is unknown.
|
|
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
|
|
perToken := h.KVBytesPerToken()
|
|
if perToken == 0 || weights <= 0 {
|
|
return 0
|
|
}
|
|
overhead := int64(1<<30) + weights/20
|
|
return weights + int64(ctxPool)*perToken + overhead
|
|
}
|
|
|
|
// ErrNotGGUF means the bytes are not a GGUF file.
|
|
var ErrNotGGUF = errors.New("not a GGUF file")
|
|
|
|
// Value types, from the GGUF specification.
|
|
const (
|
|
tUint8 = iota
|
|
tInt8
|
|
tUint16
|
|
tInt16
|
|
tUint32
|
|
tInt32
|
|
tFloat32
|
|
tBool
|
|
tString
|
|
tArray
|
|
tUint64
|
|
tInt64
|
|
tFloat64
|
|
)
|
|
|
|
// Read parses a header up to general.architecture and general.name.
|
|
func Read(r io.Reader) (Header, error) {
|
|
br := bufio.NewReader(r)
|
|
var h Header
|
|
magic := make([]byte, 4)
|
|
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
|
|
return h, ErrNotGGUF
|
|
}
|
|
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
|
|
return h, err
|
|
}
|
|
if h.Version < 1 || h.Version > 3 {
|
|
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
|
|
}
|
|
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
|
|
count := func() (uint64, error) {
|
|
if wide {
|
|
var n uint64
|
|
return n, binary.Read(br, binary.LittleEndian, &n)
|
|
}
|
|
var n uint32
|
|
err := binary.Read(br, binary.LittleEndian, &n)
|
|
return uint64(n), err
|
|
}
|
|
str := func() (string, error) {
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > 1<<20 {
|
|
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
|
|
}
|
|
b := make([]byte, n)
|
|
_, err = io.ReadFull(br, b)
|
|
return string(b), err
|
|
}
|
|
skip := func(n int64) error {
|
|
_, err := io.CopyN(io.Discard, br, n)
|
|
return err
|
|
}
|
|
// num holds the last integer value read, for the shape keys.
|
|
var num uint64
|
|
readUint := func(size int) error {
|
|
b := make([]byte, size)
|
|
if _, err := io.ReadFull(br, b); err != nil {
|
|
return err
|
|
}
|
|
switch size {
|
|
case 1:
|
|
num = uint64(b[0])
|
|
case 2:
|
|
num = uint64(binary.LittleEndian.Uint16(b))
|
|
case 4:
|
|
num = uint64(binary.LittleEndian.Uint32(b))
|
|
default:
|
|
num = binary.LittleEndian.Uint64(b)
|
|
}
|
|
return nil
|
|
}
|
|
var value func(t uint32) (string, error)
|
|
value = func(t uint32) (string, error) {
|
|
num = 0
|
|
switch t {
|
|
case tUint8, tInt8:
|
|
return "", readUint(1)
|
|
case tBool:
|
|
return "", skip(1)
|
|
case tUint16, tInt16:
|
|
return "", readUint(2)
|
|
case tUint32, tInt32:
|
|
return "", readUint(4)
|
|
case tFloat32:
|
|
return "", skip(4)
|
|
case tUint64, tInt64:
|
|
return "", readUint(8)
|
|
case tFloat64:
|
|
return "", skip(8)
|
|
case tString:
|
|
return str()
|
|
case tArray:
|
|
var et uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
|
|
return "", err
|
|
}
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > math.MaxInt32 {
|
|
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
|
|
}
|
|
for i := uint64(0); i < n; i++ {
|
|
if _, err := value(et); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
return "", fmt.Errorf("GGUF value type %d is unknown", t)
|
|
}
|
|
|
|
if _, err := count(); err != nil { // tensors
|
|
return h, err
|
|
}
|
|
kvs, err := count()
|
|
if err != nil {
|
|
return h, err
|
|
}
|
|
// Once the architecture is known, running out of the fetched bytes is
|
|
// not a failure: what the caller came for is in hand, and the shape keys
|
|
// are a bonus that a file with its tokenizer first may not reach.
|
|
partial := func(err error) (Header, error) {
|
|
if h.Architecture != "" {
|
|
return h, nil
|
|
}
|
|
return h, err
|
|
}
|
|
for i := uint64(0); i < kvs; i++ {
|
|
key, err := str()
|
|
if err != nil {
|
|
return partial(err)
|
|
}
|
|
var t uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
|
return partial(err)
|
|
}
|
|
v, err := value(t)
|
|
if err != nil {
|
|
return partial(err)
|
|
}
|
|
switch key {
|
|
case "general.architecture":
|
|
h.Architecture = v
|
|
case "general.name":
|
|
h.Name = v
|
|
}
|
|
if h.Architecture != "" {
|
|
switch strings.TrimPrefix(key, h.Architecture+".") {
|
|
case "block_count":
|
|
h.BlockCount = num
|
|
case "attention.head_count":
|
|
h.HeadCount = num
|
|
case "attention.head_count_kv":
|
|
h.HeadCountKV = num
|
|
case "embedding_length":
|
|
h.EmbeddingLength = num
|
|
case "attention.key_length":
|
|
h.KeyLength = num
|
|
case "attention.value_length":
|
|
h.ValueLength = num
|
|
}
|
|
}
|
|
// The tokenizer's arrays — megabytes of vocabulary — come after the
|
|
// architecture's keys, and nothing past them is wanted.
|
|
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
|
|
break
|
|
}
|
|
}
|
|
if h.Architecture == "" {
|
|
return h, errors.New("the GGUF header names no architecture")
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// headerBytes is how much of a file is fetched to read its header. The
|
|
// architecture is among the first keys; the tokenizer's arrays, which can be
|
|
// megabytes, come after it.
|
|
const headerBytes = 8 << 20
|
|
|
|
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
|
|
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 30 * time.Second}
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
|
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
|
}
|
|
return Read(io.LimitReader(resp.Body, headerBytes))
|
|
}
|
|
|
|
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
|
|
func IsGGUF(source string) bool {
|
|
s := strings.ToLower(strings.TrimSuffix(source, "/"))
|
|
return strings.HasSuffix(s, ".gguf")
|
|
}
|
|
|
|
// Check reads a source's header and judges it against a llama.cpp build.
|
|
//
|
|
// Only a definite answer refuses: a file that is not GGUF, or whose
|
|
// architecture the build cannot load. A header that cannot be fetched — the
|
|
// network, a directory of shards — is not a reason to refuse, so the verdict
|
|
// then says it was not checked.
|
|
func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
|
if strings.HasSuffix(url, "/") {
|
|
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
defer cancel()
|
|
h, err := FromURL(ctx, nil, url)
|
|
if errors.Is(err, ErrNotGGUF) {
|
|
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
|
|
}
|
|
if err != nil {
|
|
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
|
}
|
|
v := LlamaCpp(llamaVersion, h.Architecture)
|
|
v.Header = h
|
|
return v
|
|
}
|