// Package gguf reads what a GGUF file says about itself, from its header. // // Enough to ask one question before anything is downloaded or served: can the // engine load this? A file downloaded, catalogued and placed, and then refused // by llama.cpp with "unknown model architecture", costs a download and a // replica failing on every node it is tried on — and the answer was in the // first few kilobytes all along. package gguf import ( "bufio" "context" "encoding/binary" "errors" "fmt" "io" "math" "net/http" "strings" "time" ) // Header is the part of a GGUF header this package reads. type Header struct { Version uint32 Architecture string Name string // Type is general.type: "model", or "adapter" for a LoRA adapter, which // is applied to a model rather than run on its own. Empty in files made // before the key existed, which are models. Type string // The shape the KV cache is sized from, from the architecture's own keys // (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the // file does not say, which a caller has to treat as unknown. BlockCount uint64 HeadCount uint64 HeadCountKV uint64 EmbeddingLength uint64 // KeyLength and ValueLength are the per-head sizes when the file gives // them; otherwise a head is EmbeddingLength / HeadCount. KeyLength uint64 ValueLength uint64 } // KVBytesPerToken is what one token of context costs in llama.cpp's default // f16 cache, or 0 when the header did not give the shape. // // Keys and values, per layer, per KV head — the KV heads rather than the // attention heads, because with grouped-query attention a model with forty // heads may keep only eight, and the difference is a factor of five. func (h Header) KVBytesPerToken() int64 { kvHeads := h.HeadCountKV if kvHeads == 0 { kvHeads = h.HeadCount } if h.BlockCount == 0 || kvHeads == 0 { return 0 } k, v := h.KeyLength, h.ValueLength if k == 0 && h.HeadCount > 0 { k = h.EmbeddingLength / h.HeadCount } if v == 0 { v = k } if k == 0 { return 0 } const f16 = 2 return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers } // MemoryBytes is what one llama.cpp server needs for this model: the weights, // a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size, // which --parallel divides among its slots rather than multiplies), and room // for compute buffers and the server itself. 0 when the shape is unknown. func (h Header) MemoryBytes(weights int64, ctxPool int) int64 { perToken := h.KVBytesPerToken() if perToken == 0 || weights <= 0 { return 0 } overhead := int64(1<<30) + weights/20 return weights + int64(ctxPool)*perToken + overhead } // ErrNotGGUF means the bytes are not a GGUF file. var ErrNotGGUF = errors.New("not a GGUF file") // Value types, from the GGUF specification. const ( tUint8 = iota tInt8 tUint16 tInt16 tUint32 tInt32 tFloat32 tBool tString tArray tUint64 tInt64 tFloat64 ) // Read parses a header up to general.architecture and general.name. func Read(r io.Reader) (Header, error) { br := bufio.NewReader(r) var h Header magic := make([]byte, 4) if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" { return h, ErrNotGGUF } if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil { return h, err } if h.Version < 1 || h.Version > 3 { return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version) } wide := h.Version >= 2 // v1 used 32-bit counts and lengths count := func() (uint64, error) { if wide { var n uint64 return n, binary.Read(br, binary.LittleEndian, &n) } var n uint32 err := binary.Read(br, binary.LittleEndian, &n) return uint64(n), err } str := func() (string, error) { n, err := count() if err != nil { return "", err } if n > 1<<20 { return "", fmt.Errorf("a %d-byte string in a GGUF header", n) } b := make([]byte, n) _, err = io.ReadFull(br, b) return string(b), err } skip := func(n int64) error { _, err := io.CopyN(io.Discard, br, n) return err } // num holds the last integer value read, for the shape keys. var num uint64 readUint := func(size int) error { b := make([]byte, size) if _, err := io.ReadFull(br, b); err != nil { return err } switch size { case 1: num = uint64(b[0]) case 2: num = uint64(binary.LittleEndian.Uint16(b)) case 4: num = uint64(binary.LittleEndian.Uint32(b)) default: num = binary.LittleEndian.Uint64(b) } return nil } // depth is how many arrays the value being read is inside. Real files // nest none; a crafted one nesting thousands deep made a recursion as // deep, a few bytes per level. depth := 0 var value func(t uint32) (string, error) value = func(t uint32) (string, error) { num = 0 switch t { case tUint8, tInt8: return "", readUint(1) case tBool: return "", skip(1) case tUint16, tInt16: return "", readUint(2) case tUint32, tInt32: return "", readUint(4) case tFloat32: return "", skip(4) case tUint64, tInt64: return "", readUint(8) case tFloat64: return "", skip(8) case tString: return str() case tArray: var et uint32 if err := binary.Read(br, binary.LittleEndian, &et); err != nil { return "", err } n, err := count() if err != nil { return "", err } if n > math.MaxInt32 { return "", fmt.Errorf("a %d-element array in a GGUF header", n) } if depth >= maxArrayDepth { return "", fmt.Errorf("arrays nested more than %d deep in a GGUF header", maxArrayDepth) } depth++ defer func() { depth-- }() for i := uint64(0); i < n; i++ { if _, err := value(et); err != nil { return "", err } } return "", nil } return "", fmt.Errorf("GGUF value type %d is unknown", t) } if _, err := count(); err != nil { // tensors return h, err } kvs, err := count() if err != nil { return h, err } // Once the architecture is known, running out of the fetched bytes is // not a failure: what the caller came for is in hand, and the shape keys // are a bonus that a file with its tokenizer first may not reach. partial := func(err error) (Header, error) { if h.Architecture != "" { return h, nil } return h, err } for i := uint64(0); i < kvs; i++ { key, err := str() if err != nil { return partial(err) } var t uint32 if err := binary.Read(br, binary.LittleEndian, &t); err != nil { return partial(err) } v, err := value(t) if err != nil { return partial(err) } switch key { case "general.architecture": h.Architecture = v case "general.name": h.Name = v case "general.type": h.Type = v } if h.Architecture != "" { switch strings.TrimPrefix(key, h.Architecture+".") { case "block_count": h.BlockCount = num case "attention.head_count": h.HeadCount = num case "attention.head_count_kv": h.HeadCountKV = num case "embedding_length": h.EmbeddingLength = num case "attention.key_length": h.KeyLength = num case "attention.value_length": h.ValueLength = num } } // The tokenizer's arrays — megabytes of vocabulary — come after the // architecture's keys, and nothing past them is wanted. if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 { break } } if h.Architecture == "" { return h, errors.New("the GGUF header names no architecture") } return h, nil } // headerBytes is how much of a file is fetched to read its header. The // architecture is among the first keys; the tokenizer's arrays, which can be // megabytes, come after it. const headerBytes = 8 << 20 // FromURL reads a GGUF header over HTTP, fetching only the start of the file. func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) { if client == nil { client = &http.Client{Timeout: 30 * time.Second} } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return Header{}, err } req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1)) resp, err := client.Do(req) if err != nil { return Header{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return Header{}, fmt.Errorf("%s answered %s", url, resp.Status) } return Read(io.LimitReader(resp.Body, headerBytes)) } // maxArrayDepth is how deep arrays may nest in a header. const maxArrayDepth = 4 // IsAdapter reports whether the file is a LoRA adapter rather than a model. func (h Header) IsAdapter() bool { return h.Type == "adapter" } // adapterProbe is how much of a file is read to tell an adapter from a model: // general.type comes before the tokenizer, which is what makes a model's // header megabytes long. const adapterProbe = 256 << 10 // KindFromURL reads just enough of a GGUF file to say what it is — its type // and architecture — over HTTP. A header cut short by the probe is fine: the // keys asked for come first. func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) { if client == nil { client = &http.Client{Timeout: 30 * time.Second} } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return Header{}, err } req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1)) resp, err := client.Do(req) if err != nil { return Header{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return Header{}, fmt.Errorf("%s answered %s", url, resp.Status) } h, err := Read(io.LimitReader(resp.Body, adapterProbe)) if h.Architecture != "" { return h, nil // what was asked for was read, whatever came after } return h, err } // IsGGUF reports whether a source names a GGUF file, or a directory of them. func IsGGUF(source string) bool { s := strings.ToLower(strings.TrimSuffix(source, "/")) return strings.HasSuffix(s, ".gguf") } // Check reads a source's header and judges it against a llama.cpp build. // // Only a definite answer refuses: a file that is not GGUF, or whose // architecture the build cannot load. A header that cannot be fetched — the // network, a directory of shards — is not a reason to refuse, so the verdict // then says it was not checked. func Check(ctx context.Context, url, llamaVersion string) Verdict { if strings.HasSuffix(url, "/") { return Verdict{Servable: true, Reason: "a set of files; its header was not read"} } ctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() h, err := FromURL(ctx, nil, url) if errors.Is(err, ErrNotGGUF) { return Verdict{Checked: true, Reason: "it is not a GGUF file"} } if err != nil { return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()} } v := LlamaCpp(llamaVersion, h.Architecture) v.Header = h return v }