Files
llmgw/internal/gguf/gguf.go
T

375 lines
11 KiB
Go

// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
// Type is general.type: "model", or "adapter" for a LoRA adapter, which
// is applied to a model rather than run on its own. Empty in files made
// before the key existed, which are models.
Type string
// The shape the KV cache is sized from, from the architecture's own keys
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
// file does not say, which a caller has to treat as unknown.
BlockCount uint64
HeadCount uint64
HeadCountKV uint64
EmbeddingLength uint64
// KeyLength and ValueLength are the per-head sizes when the file gives
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
// f16 cache, or 0 when the header did not give the shape.
//
// Keys and values, per layer, per KV head — the KV heads rather than the
// attention heads, because with grouped-query attention a model with forty
// heads may keep only eight, and the difference is a factor of five.
func (h Header) KVBytesPerToken() int64 {
kvHeads := h.HeadCountKV
if kvHeads == 0 {
kvHeads = h.HeadCount
}
if h.BlockCount == 0 || kvHeads == 0 {
return 0
}
k, v := h.KeyLength, h.ValueLength
if k == 0 && h.HeadCount > 0 {
k = h.EmbeddingLength / h.HeadCount
}
if v == 0 {
v = k
}
if k == 0 {
return 0
}
const f16 = 2
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
}
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
// which --parallel divides among its slots rather than multiplies), and room
// for compute buffers and the server itself. 0 when the shape is unknown.
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
perToken := h.KVBytesPerToken()
if perToken == 0 || weights <= 0 {
return 0
}
overhead := int64(1<<30) + weights/20
return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
// num holds the last integer value read, for the shape keys.
var num uint64
readUint := func(size int) error {
b := make([]byte, size)
if _, err := io.ReadFull(br, b); err != nil {
return err
}
switch size {
case 1:
num = uint64(b[0])
case 2:
num = uint64(binary.LittleEndian.Uint16(b))
case 4:
num = uint64(binary.LittleEndian.Uint32(b))
default:
num = binary.LittleEndian.Uint64(b)
}
return nil
}
// depth is how many arrays the value being read is inside. Real files
// nest none; a crafted one nesting thousands deep made a recursion as
// deep, a few bytes per level.
depth := 0
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
num = 0
switch t {
case tUint8, tInt8:
return "", readUint(1)
case tBool:
return "", skip(1)
case tUint16, tInt16:
return "", readUint(2)
case tUint32, tInt32:
return "", readUint(4)
case tFloat32:
return "", skip(4)
case tUint64, tInt64:
return "", readUint(8)
case tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
if depth >= maxArrayDepth {
return "", fmt.Errorf("arrays nested more than %d deep in a GGUF header", maxArrayDepth)
}
depth++
defer func() { depth-- }()
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
// Once the architecture is known, running out of the fetched bytes is
// not a failure: what the caller came for is in hand, and the shape keys
// are a bonus that a file with its tokenizer first may not reach.
partial := func(err error) (Header, error) {
if h.Architecture != "" {
return h, nil
}
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return partial(err)
}
v, err := value(t)
if err != nil {
return partial(err)
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
case "general.type":
h.Type = v
}
if h.Architecture != "" {
switch strings.TrimPrefix(key, h.Architecture+".") {
case "block_count":
h.BlockCount = num
case "attention.head_count":
h.HeadCount = num
case "attention.head_count_kv":
h.HeadCountKV = num
case "embedding_length":
h.EmbeddingLength = num
case "attention.key_length":
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
// architecture's keys, and nothing past them is wanted.
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// maxArrayDepth is how deep arrays may nest in a header.
const maxArrayDepth = 4
// IsAdapter reports whether the file is a LoRA adapter rather than a model.
func (h Header) IsAdapter() bool { return h.Type == "adapter" }
// adapterProbe is how much of a file is read to tell an adapter from a model:
// general.type comes before the tokenizer, which is what makes a model's
// header megabytes long.
const adapterProbe = 256 << 10
// KindFromURL reads just enough of a GGUF file to say what it is — its type
// and architecture — over HTTP. A header cut short by the probe is fine: the
// keys asked for come first.
func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
h, err := Read(io.LimitReader(resp.Body, adapterProbe))
if h.Architecture != "" {
return h, nil // what was asked for was read, whatever came after
}
return h, err
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
v := LlamaCpp(llamaVersion, h.Architecture)
v.Header = h
return v
}