Files
llmgw/internal/gguf/gguf.go
T
Ilker ManapandClaude Opus 5.5 c22528c26e
CI / test (push) Failing after 30m3s
A model that does not fit its card is not added to fail; weights in use are not removed
Apertus 70B with a 65536-token pool, and then 8B with 131072, on a 24 GB
card loaded their weights and failed to make their context on every start,
the 8B taking the card from the 70B each time only to fail. ctl model serve
now shrinks a pool the card cannot hold beside the weights (KV cache from
the GGUF's shape, 90% of the card, a gigabyte for the engine), saying so,
and refuses weights that do not fit at all; the portal's serve form refuses
such a pool and says the largest that fits.

The mirror page removed the 70B's weights while the catalog still had the
model, whose replicas went on loading the node's copy with nothing said.
Removing weights a catalog model loads is refused, naming the model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 00:22:01 +02:00

418 lines
12 KiB
Go

// Package gguf reads what a GGUF file says about itself, from its header.
//
// Enough to ask one question before anything is downloaded or served: can the
// engine load this? A file downloaded, catalogued and placed, and then refused
// by llama.cpp with "unknown model architecture", costs a download and a
// replica failing on every node it is tried on — and the answer was in the
// first few kilobytes all along.
package gguf
import (
"bufio"
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
)
// Header is the part of a GGUF header this package reads.
type Header struct {
Version uint32
Architecture string
Name string
// Type is general.type: "model", or "adapter" for a LoRA adapter, which
// is applied to a model rather than run on its own. Empty in files made
// before the key existed, which are models.
Type string
// The shape the KV cache is sized from, from the architecture's own keys
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
// file does not say, which a caller has to treat as unknown.
BlockCount uint64
HeadCount uint64
HeadCountKV uint64
EmbeddingLength uint64
// KeyLength and ValueLength are the per-head sizes when the file gives
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
// Embedding is whether the model pools its tokens into one vector
// (<arch>.pooling_type set and not none): an embedding model, which
// llama.cpp serves with --embedding and does not chat.
Embedding bool
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
// f16 cache, or 0 when the header did not give the shape.
//
// Keys and values, per layer, per KV head — the KV heads rather than the
// attention heads, because with grouped-query attention a model with forty
// heads may keep only eight, and the difference is a factor of five.
func (h Header) KVBytesPerToken() int64 {
kvHeads := h.HeadCountKV
if kvHeads == 0 {
kvHeads = h.HeadCount
}
if h.BlockCount == 0 || kvHeads == 0 {
return 0
}
k, v := h.KeyLength, h.ValueLength
if k == 0 && h.HeadCount > 0 {
k = h.EmbeddingLength / h.HeadCount
}
if v == 0 {
v = k
}
if k == 0 {
return 0
}
const f16 = 2
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
}
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
// which --parallel divides among its slots rather than multiplies), and room
// for compute buffers and the server itself. 0 when the shape is unknown.
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
perToken := h.KVBytesPerToken()
if perToken == 0 || weights <= 0 {
return 0
}
overhead := int64(1<<30) + weights/20
return weights + int64(ctxPool)*perToken + overhead
}
// ErrNotGGUF means the bytes are not a GGUF file.
var ErrNotGGUF = errors.New("not a GGUF file")
// Value types, from the GGUF specification.
const (
tUint8 = iota
tInt8
tUint16
tInt16
tUint32
tInt32
tFloat32
tBool
tString
tArray
tUint64
tInt64
tFloat64
)
// Read parses a header up to general.architecture and general.name.
func Read(r io.Reader) (Header, error) {
br := bufio.NewReader(r)
var h Header
magic := make([]byte, 4)
if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" {
return h, ErrNotGGUF
}
if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil {
return h, err
}
if h.Version < 1 || h.Version > 3 {
return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version)
}
wide := h.Version >= 2 // v1 used 32-bit counts and lengths
count := func() (uint64, error) {
if wide {
var n uint64
return n, binary.Read(br, binary.LittleEndian, &n)
}
var n uint32
err := binary.Read(br, binary.LittleEndian, &n)
return uint64(n), err
}
str := func() (string, error) {
n, err := count()
if err != nil {
return "", err
}
if n > 1<<20 {
return "", fmt.Errorf("a %d-byte string in a GGUF header", n)
}
b := make([]byte, n)
_, err = io.ReadFull(br, b)
return string(b), err
}
skip := func(n int64) error {
_, err := io.CopyN(io.Discard, br, n)
return err
}
// num holds the last integer value read, for the shape keys.
var num uint64
readUint := func(size int) error {
b := make([]byte, size)
if _, err := io.ReadFull(br, b); err != nil {
return err
}
switch size {
case 1:
num = uint64(b[0])
case 2:
num = uint64(binary.LittleEndian.Uint16(b))
case 4:
num = uint64(binary.LittleEndian.Uint32(b))
default:
num = binary.LittleEndian.Uint64(b)
}
return nil
}
// depth is how many arrays the value being read is inside. Real files
// nest none; a crafted one nesting thousands deep made a recursion as
// deep, a few bytes per level.
depth := 0
var value func(t uint32) (string, error)
value = func(t uint32) (string, error) {
num = 0
switch t {
case tUint8, tInt8:
return "", readUint(1)
case tBool:
return "", skip(1)
case tUint16, tInt16:
return "", readUint(2)
case tUint32, tInt32:
return "", readUint(4)
case tFloat32:
return "", skip(4)
case tUint64, tInt64:
return "", readUint(8)
case tFloat64:
return "", skip(8)
case tString:
return str()
case tArray:
var et uint32
if err := binary.Read(br, binary.LittleEndian, &et); err != nil {
return "", err
}
n, err := count()
if err != nil {
return "", err
}
if n > math.MaxInt32 {
return "", fmt.Errorf("a %d-element array in a GGUF header", n)
}
if depth >= maxArrayDepth {
return "", fmt.Errorf("arrays nested more than %d deep in a GGUF header", maxArrayDepth)
}
depth++
defer func() { depth-- }()
for i := uint64(0); i < n; i++ {
if _, err := value(et); err != nil {
return "", err
}
}
return "", nil
}
return "", fmt.Errorf("GGUF value type %d is unknown", t)
}
if _, err := count(); err != nil { // tensors
return h, err
}
kvs, err := count()
if err != nil {
return h, err
}
// Once the architecture is known, running out of the fetched bytes is
// not a failure: what the caller came for is in hand, and the shape keys
// are a bonus that a file with its tokenizer first may not reach.
partial := func(err error) (Header, error) {
if h.Architecture != "" {
return h, nil
}
return h, err
}
for i := uint64(0); i < kvs; i++ {
key, err := str()
if err != nil {
return partial(err)
}
var t uint32
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
return partial(err)
}
v, err := value(t)
if err != nil {
return partial(err)
}
switch key {
case "general.architecture":
h.Architecture = v
case "general.name":
h.Name = v
case "general.type":
h.Type = v
}
if h.Architecture != "" {
switch strings.TrimPrefix(key, h.Architecture+".") {
case "block_count":
h.BlockCount = num
case "attention.head_count":
h.HeadCount = num
case "attention.head_count_kv":
h.HeadCountKV = num
case "embedding_length":
h.EmbeddingLength = num
case "attention.key_length":
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
case "pooling_type":
h.Embedding = num > 0
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
// architecture's keys, and nothing past them is wanted.
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
break
}
}
if h.Architecture == "" {
return h, errors.New("the GGUF header names no architecture")
}
return h, nil
}
// headerBytes is how much of a file is fetched to read its header. The
// architecture is among the first keys; the tokenizer's arrays, which can be
// megabytes, come after it.
const headerBytes = 8 << 20
// FromURL reads a GGUF header over HTTP, fetching only the start of the file.
func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
return Read(io.LimitReader(resp.Body, headerBytes))
}
// maxArrayDepth is how deep arrays may nest in a header.
const maxArrayDepth = 4
// IsAdapter reports whether the file is a LoRA adapter rather than a model.
func (h Header) IsAdapter() bool { return h.Type == "adapter" }
// adapterProbe is how much of a file is read to tell an adapter from a model:
// general.type comes before the tokenizer, which is what makes a model's
// header megabytes long.
const adapterProbe = 256 << 10
// KindFromURL reads just enough of a GGUF file to say what it is — its type
// and architecture — over HTTP. A header cut short by the probe is fine: the
// keys asked for come first.
func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
if client == nil {
client = &http.Client{Timeout: 30 * time.Second}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Header{}, err
}
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1))
resp, err := client.Do(req)
if err != nil {
return Header{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
}
h, err := Read(io.LimitReader(resp.Body, adapterProbe))
if h.Architecture != "" {
return h, nil // what was asked for was read, whatever came after
}
return h, err
}
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
func IsGGUF(source string) bool {
s := strings.ToLower(strings.TrimSuffix(source, "/"))
return strings.HasSuffix(s, ".gguf")
}
// Check reads a source's header and judges it against a llama.cpp build.
//
// Only a definite answer refuses: a file that is not GGUF, or whose
// architecture the build cannot load. A header that cannot be fetched — the
// network, a directory of shards — is not a reason to refuse, so the verdict
// then says it was not checked.
func Check(ctx context.Context, url, llamaVersion string) Verdict {
if strings.HasSuffix(url, "/") {
return Verdict{Servable: true, Reason: "a set of files; its header was not read"}
}
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
h, err := FromURL(ctx, nil, url)
if errors.Is(err, ErrNotGGUF) {
return Verdict{Checked: true, Reason: "it is not a GGUF file"}
}
if err != nil {
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
}
v := LlamaCpp(llamaVersion, h.Architecture)
v.Header = h
return v
}
// CardUsable is the share of a card's memory a model may take: the rest is
// CUDA's own and the engine's compute buffers.
const CardUsable = 0.9
// PoolThatFits is the largest context pool, in tokens, whose KV cache fits
// on gpus cards of cardGB beside weights bytes of weights and a gigabyte for
// the engine: -1 when the weights alone do not, 0 when the header or the
// card does not say enough to tell.
func (h Header) PoolThatFits(weights int64, cardGB, gpus int) int {
perTok := h.KVBytesPerToken()
if cardGB <= 0 || perTok <= 0 || weights <= 0 {
return 0
}
budget := float64(cardGB*max(gpus, 1))*CardUsable*(1<<30) - float64(weights) - (1 << 30)
if budget <= 0 {
return -1
}
return int(budget / float64(perTok))
}
// ClassMemGB is a card's memory as its class names it: 6000-24g is 24, 0
// for a class that does not say.
func ClassMemGB(class string) int {
i := strings.LastIndex(class, "-")
if i < 0 || !strings.HasSuffix(class, "g") {
return 0
}
n := 0
for _, c := range strings.TrimSuffix(class[i+1:], "g") {
if c < '0' || c > '9' {
return 0
}
n = n*10 + int(c-'0')
}
return n
}