// Package gguf reads what a GGUF file says about itself, from its header. // // Enough to ask one question before anything is downloaded or served: can the // engine load this? A file downloaded, catalogued and placed, and then refused // by llama.cpp with "unknown model architecture", costs a download and a // replica failing on every node it is tried on — and the answer was in the // first few kilobytes all along. package gguf import ( "bufio" "context" "encoding/binary" "errors" "fmt" "io" "math" "net/http" "strings" "time" ) // Header is the part of a GGUF header this package reads. type Header struct { Version uint32 Architecture string Name string } // ErrNotGGUF means the bytes are not a GGUF file. var ErrNotGGUF = errors.New("not a GGUF file") // Value types, from the GGUF specification. const ( tUint8 = iota tInt8 tUint16 tInt16 tUint32 tInt32 tFloat32 tBool tString tArray tUint64 tInt64 tFloat64 ) // Read parses a header up to general.architecture and general.name. func Read(r io.Reader) (Header, error) { br := bufio.NewReader(r) var h Header magic := make([]byte, 4) if _, err := io.ReadFull(br, magic); err != nil || string(magic) != "GGUF" { return h, ErrNotGGUF } if err := binary.Read(br, binary.LittleEndian, &h.Version); err != nil { return h, err } if h.Version < 1 || h.Version > 3 { return h, fmt.Errorf("GGUF version %d is not one this reads", h.Version) } wide := h.Version >= 2 // v1 used 32-bit counts and lengths count := func() (uint64, error) { if wide { var n uint64 return n, binary.Read(br, binary.LittleEndian, &n) } var n uint32 err := binary.Read(br, binary.LittleEndian, &n) return uint64(n), err } str := func() (string, error) { n, err := count() if err != nil { return "", err } if n > 1<<20 { return "", fmt.Errorf("a %d-byte string in a GGUF header", n) } b := make([]byte, n) _, err = io.ReadFull(br, b) return string(b), err } skip := func(n int64) error { _, err := io.CopyN(io.Discard, br, n) return err } var value func(t uint32) (string, error) value = func(t uint32) (string, error) { switch t { case tUint8, tInt8, tBool: return "", skip(1) case tUint16, tInt16: return "", skip(2) case tUint32, tInt32, tFloat32: return "", skip(4) case tUint64, tInt64, tFloat64: return "", skip(8) case tString: return str() case tArray: var et uint32 if err := binary.Read(br, binary.LittleEndian, &et); err != nil { return "", err } n, err := count() if err != nil { return "", err } if n > math.MaxInt32 { return "", fmt.Errorf("a %d-element array in a GGUF header", n) } for i := uint64(0); i < n; i++ { if _, err := value(et); err != nil { return "", err } } return "", nil } return "", fmt.Errorf("GGUF value type %d is unknown", t) } if _, err := count(); err != nil { // tensors return h, err } kvs, err := count() if err != nil { return h, err } for i := uint64(0); i < kvs; i++ { key, err := str() if err != nil { return h, err } var t uint32 if err := binary.Read(br, binary.LittleEndian, &t); err != nil { return h, err } v, err := value(t) if err != nil { return h, err } switch key { case "general.architecture": h.Architecture = v case "general.name": h.Name = v } if h.Architecture != "" && h.Name != "" { break } } if h.Architecture == "" { return h, errors.New("the GGUF header names no architecture") } return h, nil } // headerBytes is how much of a file is fetched to read its header. The // architecture is among the first keys; the tokenizer's arrays, which can be // megabytes, come after it. const headerBytes = 8 << 20 // FromURL reads a GGUF header over HTTP, fetching only the start of the file. func FromURL(ctx context.Context, client *http.Client, url string) (Header, error) { if client == nil { client = &http.Client{Timeout: 30 * time.Second} } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return Header{}, err } req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", headerBytes-1)) resp, err := client.Do(req) if err != nil { return Header{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return Header{}, fmt.Errorf("%s answered %s", url, resp.Status) } return Read(io.LimitReader(resp.Body, headerBytes)) } // IsGGUF reports whether a source names a GGUF file, or a directory of them. func IsGGUF(source string) bool { s := strings.ToLower(strings.TrimSuffix(source, "/")) return strings.HasSuffix(s, ".gguf") } // Check reads a source's header and judges it against a llama.cpp build. // // Only a definite answer refuses: a file that is not GGUF, or whose // architecture the build cannot load. A header that cannot be fetched — the // network, a directory of shards — is not a reason to refuse, so the verdict // then says it was not checked. func Check(ctx context.Context, url, llamaVersion string) Verdict { if strings.HasSuffix(url, "/") { return Verdict{Servable: true, Reason: "a set of files; its header was not read"} } ctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() h, err := FromURL(ctx, nil, url) if errors.Is(err, ErrNotGGUF) { return Verdict{Checked: true, Reason: "it is not a GGUF file"} } if err != nil { return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()} } return LlamaCpp(llamaVersion, h.Architecture) }