package gguf import ( "bytes" "context" "encoding/binary" "errors" "net/http" "net/http/httptest" "strings" "testing" "time" ) // header builds a GGUF v3 header: an array before the architecture, the way // real files put tokenizer data, so skipping values is exercised. func header(arch string) []byte { var b bytes.Buffer w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } str := func(s string) { w(uint64(len(s))); b.WriteString(s) } b.WriteString("GGUF") w(uint32(3)) w(uint64(0)) // tensors w(uint64(4)) // kvs str("general.alignment") w(uint32(tUint32)) w(uint32(32)) str("some.list") w(uint32(tArray)) w(uint32(tString)) w(uint64(2)) str("a") str("bc") str("general.architecture") w(uint32(tString)) str(arch) str("general.name") w(uint32(tString)) str("A Model") return b.Bytes() } func TestTheArchitectureIsReadFromTheHeader(t *testing.T) { h, err := Read(bytes.NewReader(header("ggmlc"))) if err != nil { t.Fatal(err) } if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 { t.Errorf("header = %+v", h) } } func TestSomethingElseIsNotAGGUF(t *testing.T) { if _, err := Read(strings.NewReader("not found")); !errors.Is(err, ErrNotGGUF) { t.Errorf("an HTML page read as %v", err) } } func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) { body := append(header("qwen2"), make([]byte, 1<<20)...) var rng string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rng = r.Header.Get("Range") http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body)) })) defer srv.Close() h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf") if err != nil || h.Architecture != "qwen2" { t.Fatalf("%+v, %v", h, err) } if !strings.HasPrefix(rng, "bytes=0-") { t.Errorf("asked for %q rather than the start of the file", rng) } } func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) { if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked { t.Errorf("qwen2: %+v", v) } // The file that started this: downloaded, catalogued, placed, and then // refused by every node with "unknown model architecture: 'ggmlc'". if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") { t.Errorf("ggmlc: %+v", v) } if v := LlamaCpp("b10991", "clip"); v.Servable { t.Errorf("a vision projector alone: %+v", v) } // A build llmgw has no list for is not refused, only not vouched for. if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked { t.Errorf("an unknown build: %+v", v) } } var timeZero = time.Time{} // shapedHeader is a header with the keys a KV cache is sized from, written // with the integer types llama.cpp uses for them. func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte { var b bytes.Buffer w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } str := func(s string) { w(uint64(len(s))); b.WriteString(s) } u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) } b.WriteString("GGUF") w(uint32(3)) w(uint64(0)) w(uint64(7)) str("general.architecture") w(uint32(tString)) str(arch) u32(arch+".block_count", layers) u32(arch+".attention.head_count", heads) u32(arch+".attention.head_count_kv", kvHeads) u32(arch+".embedding_length", embd) u32(arch+".context_length", 32768) str("tokenizer.ggml.tokens") w(uint32(tArray)) w(uint32(tString)) w(uint64(1 << 20)) if !truncateTokenizer { for i := 0; i < 1<<20; i++ { str("t") } } return b.Bytes() } func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) { // Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V, // 5120 wide — 196608 bytes a token in f16, as read from the live mirror. h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true))) if err != nil { t.Fatalf("a header cut off in its tokenizer failed to read: %v", err) } if got := h.KVBytesPerToken(); got != 196608 { t.Errorf("%d bytes a token, expected 196608", got) } weights := int64(9 << 30) want := weights + 16384*196608 + (1<<30 + weights/20) if got := h.MemoryBytes(weights, 16384); got != want { t.Errorf("memory %d, expected %d", got, want) } if (Header{}).MemoryBytes(weights, 16384) != 0 { t.Error("a header with no shape produced a figure") } } func TestArraysNestedDeepAreRefused(t *testing.T) { var b bytes.Buffer w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } b.WriteString("GGUF") w(uint32(3)) w(uint64(0)) w(uint64(1)) w(uint64(len("deep"))) b.WriteString("deep") w(uint32(tArray)) for range 10_000 { w(uint32(tArray)) w(uint64(1)) } _, err := Read(bytes.NewReader(b.Bytes())) if err == nil || !strings.Contains(err.Error(), "nested") { t.Errorf("ten thousand nested arrays: %v", err) } }