Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
375 lines
11 KiB
Go
375 lines
11 KiB
Go
// Package gguf reads what a GGUF file says about itself, from its header.
|
|
//
|
|
// Enough to ask one question before anything is downloaded or served: can the
|
|
// engine load this? A file downloaded, catalogued and placed, and then refused
|
|
// by llama.cpp with "unknown model architecture", costs a download and a
|
|
// replica failing on every node it is tried on — and the answer was in the
|
|
// first few kilobytes all along.
|
|
package gguf
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Header is the part of a GGUF header this package reads.
|
|
type Header struct {
|
|
Version uint32
|
|
Architecture string
|
|
Name string
|
|
// Type is general.type: "model", or "adapter" for a LoRA adapter, which
|
|
// is applied to a model rather than run on its own. Empty in files made
|
|
// before the key existed, which are models.
|
|
Type string
|
|
|
|
// The shape the KV cache is sized from, from the architecture's own keys
|
|
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
|
|
// file does not say, which a caller has to treat as unknown.
|
|
BlockCount uint64
|
|
HeadCount uint64
|
|
HeadCountKV uint64
|
|
EmbeddingLength uint64
|
|
// KeyLength and ValueLength are the per-head sizes when the file gives
|
|
// them; otherwise a head is EmbeddingLength / HeadCount.
|
|
KeyLength uint64
|
|
ValueLength uint64
|
|
}
|
|
|
|
// KVBytesPerToken is what one token of context costs in llama.cpp's default
|
|
// f16 cache, or 0 when the header did not give the shape.
|
|
//
|
|
// Keys and values, per layer, per KV head — the KV heads rather than the
|
|
// attention heads, because with grouped-query attention a model with forty
|
|
// heads may keep only eight, and the difference is a factor of five.
|
|
func (h Header) KVBytesPerToken() int64 {
|
|
kvHeads := h.HeadCountKV
|
|
if kvHeads == 0 {
|
|
kvHeads = h.HeadCount
|
|
}
|
|
if h.BlockCount == 0 || kvHeads == 0 {
|
|
return 0
|
|
}
|
|
k, v := h.KeyLength, h.ValueLength
|
|
if k == 0 && h.HeadCount > 0 {
|
|
k = h.EmbeddingLength / h.HeadCount
|
|
}
|
|
if v == 0 {
|
|
v = k
|
|
}
|
|
if k == 0 {
|
|
return 0
|
|
}
|
|
const f16 = 2
|
|
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
|
|
}
|
|
|
|
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
|
|
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
|
|
// which --parallel divides among its slots rather than multiplies), and room
|
|
// for compute buffers and the server itself. 0 when the shape is unknown.
|
|
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
|
|
perToken := h.KVBytesPerToken()
|
|
if perToken == 0 || weights <= 0 {
|
|
return 0
|
|
}
|
|
overhead := int64(1<<30) + weights/20
|
|
return weights + int64(ctxPool)*perToken + overhead
|
|
}
|
|
|
|
// ErrNotGGUF means the bytes are not a GGUF file.
|
|
var ErrNotGGUF = errors.New("not a GGUF file")
|
|
|
|
// Value types, from the GGUF specification.
|
|
const (
|
|
tUint8 = iota
|
|
tInt8
|
|
tUint16
|
|
tInt16
|
|
tUint32
|
|
tInt32
|
|
tFloat32
|
|
tBool
|
|
tString
|
|
tArray
|
|
tUint64
|
|
tInt64
|
|
tFloat64
|
|
)
|
|
|
|
// Read parses a header up to general.architecture and general.name.
|
|
func Read(r io.Reader) (Header, error) {
|
|
br := bufio.NewReader(r)
|
|
var h Header
|
|
magic := make([]byte, 4)
|
|
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
|
|
return h, ErrNotGGUF
|
|
}
|
|
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
|
|
return h, err
|
|
}
|
|
if h.Version < 1 || h.Version > 3 {
|
|
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
|
|
}
|
|
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
|
|
count := func() (uint64, error) {
|
|
if wide {
|
|
var n uint64
|
|
return n, binary.Read(br, binary.LittleEndian, &n)
|
|
}
|
|
var n uint32
|
|
err := binary.Read(br, binary.LittleEndian, &n)
|
|
return uint64(n), err
|
|
}
|
|
str := func() (string, error) {
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > 1<<20 {
|
|
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
|
|
}
|
|
b := make([]byte, n)
|
|
_, err = io.ReadFull(br, b)
|
|
return string(b), err
|
|
}
|
|
skip := func(n int64) error {
|
|
_, err := io.CopyN(io.Discard, br, n)
|
|
return err
|
|
}
|
|
// num holds the last integer value read, for the shape keys.
|
|
var num uint64
|
|
readUint := func(size int) error {
|
|
b := make([]byte, size)
|
|
if _, err := io.ReadFull(br, b); err != nil {
|
|
return err
|
|
}
|
|
switch size {
|
|
case 1:
|
|
num = uint64(b[0])
|
|
case 2:
|
|
num = uint64(binary.LittleEndian.Uint16(b))
|
|
case 4:
|
|
num = uint64(binary.LittleEndian.Uint32(b))
|
|
default:
|
|
num = binary.LittleEndian.Uint64(b)
|
|
}
|
|
return nil
|
|
}
|
|
// depth is how many arrays the value being read is inside. Real files
|
|
// nest none; a crafted one nesting thousands deep made a recursion as
|
|
// deep, a few bytes per level.
|
|
depth := 0
|
|
var value func(t uint32) (string, error)
|
|
value = func(t uint32) (string, error) {
|
|
num = 0
|
|
switch t {
|
|
case tUint8, tInt8:
|
|
return "", readUint(1)
|
|
case tBool:
|
|
return "", skip(1)
|
|
case tUint16, tInt16:
|
|
return "", readUint(2)
|
|
case tUint32, tInt32:
|
|
return "", readUint(4)
|
|
case tFloat32:
|
|
return "", skip(4)
|
|
case tUint64, tInt64:
|
|
return "", readUint(8)
|
|
case tFloat64:
|
|
return "", skip(8)
|
|
case tString:
|
|
return str()
|
|
case tArray:
|
|
var et uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
|
|
return "", err
|
|
}
|
|
n, err := count()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if n > math.MaxInt32 {
|
|
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
|
|
}
|
|
if depth >= maxArrayDepth {
|
|
return "", fmt.Errorf("arrays nested more than %d deep in a GGUF header", maxArrayDepth)
|
|
}
|
|
depth++
|
|
defer func() { depth-- }()
|
|
for i := uint64(0); i < n; i++ {
|
|
if _, err := value(et); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
return "", fmt.Errorf("GGUF value type %d is unknown", t)
|
|
}
|
|
|
|
if _, err := count(); err != nil { // tensors
|
|
return h, err
|
|
}
|
|
kvs, err := count()
|
|
if err != nil {
|
|
return h, err
|
|
}
|
|
// Once the architecture is known, running out of the fetched bytes is
|
|
// not a failure: what the caller came for is in hand, and the shape keys
|
|
// are a bonus that a file with its tokenizer first may not reach.
|
|
partial := func(err error) (Header, error) {
|
|
if h.Architecture != "" {
|
|
return h, nil
|
|
}
|
|
return h, err
|
|
}
|
|
for i := uint64(0); i < kvs; i++ {
|
|
key, err := str()
|
|
if err != nil {
|
|
return partial(err)
|
|
}
|
|
var t uint32
|
|
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
|
return partial(err)
|
|
}
|
|
v, err := value(t)
|
|
if err != nil {
|
|
return partial(err)
|
|
}
|
|
switch key {
|
|
case "general.architecture":
|
|
h.Architecture = v
|
|
case "general.name":
|
|
h.Name = v
|
|
case "general.type":
|
|
h.Type = v
|
|
}
|
|
if h.Architecture != "" {
|
|
switch strings.TrimPrefix(key, h.Architecture+".") {
|
|
case "block_count":
|
|
h.BlockCount = num
|
|
case "attention.head_count":
|
|
h.HeadCount = num
|
|
case "attention.head_count_kv":
|
|
h.HeadCountKV = num
|
|
case "embedding_length":
|
|
h.EmbeddingLength = num
|
|
case "attention.key_length":
|
|
h.KeyLength = num
|
|
case "attention.value_length":
|
|
h.ValueLength = num
|
|
}
|
|
}
|
|
// The tokenizer's arrays — megabytes of vocabulary — come after the
|
|
// architecture's keys, and nothing past them is wanted.
|
|
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
|
|
break
|
|
}
|
|
}
|
|
if h.Architecture == "" {
|
|
return h, errors.New("the GGUF header names no architecture")
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// headerBytes is how much of a file is fetched to read its header. The
|
|
// architecture is among the first keys; the tokenizer's arrays, which can be
|
|
// megabytes, come after it.
|
|
const headerBytes = 8 << 20
|
|
|
|
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
|
|
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 30 * time.Second}
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
|
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
|
}
|
|
return Read(io.LimitReader(resp.Body, headerBytes))
|
|
}
|
|
|
|
// maxArrayDepth is how deep arrays may nest in a header.
|
|
const maxArrayDepth = 4
|
|
|
|
// IsAdapter reports whether the file is a LoRA adapter rather than a model.
|
|
func (h Header) IsAdapter() bool { return h.Type == "adapter" }
|
|
|
|
// adapterProbe is how much of a file is read to tell an adapter from a model:
|
|
// general.type comes before the tokenizer, which is what makes a model's
|
|
// header megabytes long.
|
|
const adapterProbe = 256 << 10
|
|
|
|
// KindFromURL reads just enough of a GGUF file to say what it is — its type
|
|
// and architecture — over HTTP. A header cut short by the probe is fine: the
|
|
// keys asked for come first.
|
|
func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 30 * time.Second}
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1))
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return Header{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
|
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
|
}
|
|
h, err := Read(io.LimitReader(resp.Body, adapterProbe))
|
|
if h.Architecture != "" {
|
|
return h, nil // what was asked for was read, whatever came after
|
|
}
|
|
return h, err
|
|
}
|
|
|
|
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
|
|
func IsGGUF(source string) bool {
|
|
s := strings.ToLower(strings.TrimSuffix(source, "/"))
|
|
return strings.HasSuffix(s, ".gguf")
|
|
}
|
|
|
|
// Check reads a source's header and judges it against a llama.cpp build.
|
|
//
|
|
// Only a definite answer refuses: a file that is not GGUF, or whose
|
|
// architecture the build cannot load. A header that cannot be fetched — the
|
|
// network, a directory of shards — is not a reason to refuse, so the verdict
|
|
// then says it was not checked.
|
|
func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
|
if strings.HasSuffix(url, "/") {
|
|
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
defer cancel()
|
|
h, err := FromURL(ctx, nil, url)
|
|
if errors.Is(err, ErrNotGGUF) {
|
|
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
|
|
}
|
|
if err != nil {
|
|
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
|
}
|
|
v := LlamaCpp(llamaVersion, h.Architecture)
|
|
v.Header = h
|
|
return v
|
|
}
|