Files
llmgw/internal/gguf/gguf_test.go
T

166 lines
4.7 KiB
Go

package gguf
import (
"bytes"
"context"
"encoding/binary"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// header builds a GGUF v3 header: an array before the architecture, the way
// real files put tokenizer data, so skipping values is exercised.
func header(arch string) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0)) // tensors
w(uint64(4)) // kvs
str("general.alignment")
w(uint32(tUint32))
w(uint32(32))
str("some.list")
w(uint32(tArray))
w(uint32(tString))
w(uint64(2))
str("a")
str("bc")
str("general.architecture")
w(uint32(tString))
str(arch)
str("general.name")
w(uint32(tString))
str("A Model")
return b.Bytes()
}
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
h, err := Read(bytes.NewReader(header("ggmlc")))
if err != nil {
t.Fatal(err)
}
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
t.Errorf("header = %+v", h)
}
}
func TestSomethingElseIsNotAGGUF(t *testing.T) {
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
t.Errorf("an HTML page read as %v", err)
}
}
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
body := append(header("qwen2"), make([]byte, 1<<20)...)
var rng string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rng = r.Header.Get("Range")
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
}))
defer srv.Close()
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
if err != nil || h.Architecture != "qwen2" {
t.Fatalf("%+v, %v", h, err)
}
if !strings.HasPrefix(rng, "bytes=0-") {
t.Errorf("asked for %q rather than the start of the file", rng)
}
}
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
t.Errorf("qwen2: %+v", v)
}
// The file that started this: downloaded, catalogued, placed, and then
// refused by every node with "unknown model architecture: 'ggmlc'".
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
t.Errorf("ggmlc: %+v", v)
}
if v := LlamaCpp("b10991", "clip"); v.Servable {
t.Errorf("a vision projector alone: %+v", v)
}
// A build llmgw has no list for is not refused, only not vouched for.
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
t.Errorf("an unknown build: %+v", v)
}
}
var timeZero = time.Time{}
// shapedHeader is a header with the keys a KV cache is sized from, written
// with the integer types llama.cpp uses for them.
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(7))
str("general.architecture")
w(uint32(tString))
str(arch)
u32(arch+".block_count", layers)
u32(arch+".attention.head_count", heads)
u32(arch+".attention.head_count_kv", kvHeads)
u32(arch+".embedding_length", embd)
u32(arch+".context_length", 32768)
str("tokenizer.ggml.tokens")
w(uint32(tArray))
w(uint32(tString))
w(uint64(1 << 20))
if !truncateTokenizer {
for i := 0; i < 1<<20; i++ {
str("t")
}
}
return b.Bytes()
}
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
if err != nil {
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
}
if got := h.KVBytesPerToken(); got != 196608 {
t.Errorf("%d bytes a token, expected 196608", got)
}
weights := int64(9 << 30)
want := weights + 16384*196608 + (1<<30 + weights/20)
if got := h.MemoryBytes(weights, 16384); got != want {
t.Errorf("memory %d, expected %d", got, want)
}
if (Header{}).MemoryBytes(weights, 16384) != 0 {
t.Error("a header with no shape produced a figure")
}
}
func TestArraysNestedDeepAreRefused(t *testing.T) {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(1))
w(uint64(len("deep")))
b.WriteString("deep")
w(uint32(tArray))
for range 10_000 {
w(uint32(tArray))
w(uint64(1))
}
_, err := Read(bytes.NewReader(b.Bytes()))
if err == nil || !strings.Contains(err.Error(), "nested") {
t.Errorf("ten thousand nested arrays: %v", err)
}
}