Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
166 lines
4.7 KiB
Go
166 lines
4.7 KiB
Go
package gguf
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// header builds a GGUF v3 header: an array before the architecture, the way
|
|
// real files put tokenizer data, so skipping values is exercised.
|
|
func header(arch string) []byte {
|
|
var b bytes.Buffer
|
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
|
b.WriteString("GGUF")
|
|
w(uint32(3))
|
|
w(uint64(0)) // tensors
|
|
w(uint64(4)) // kvs
|
|
str("general.alignment")
|
|
w(uint32(tUint32))
|
|
w(uint32(32))
|
|
str("some.list")
|
|
w(uint32(tArray))
|
|
w(uint32(tString))
|
|
w(uint64(2))
|
|
str("a")
|
|
str("bc")
|
|
str("general.architecture")
|
|
w(uint32(tString))
|
|
str(arch)
|
|
str("general.name")
|
|
w(uint32(tString))
|
|
str("A Model")
|
|
return b.Bytes()
|
|
}
|
|
|
|
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
|
|
h, err := Read(bytes.NewReader(header("ggmlc")))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
|
|
t.Errorf("header = %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestSomethingElseIsNotAGGUF(t *testing.T) {
|
|
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
|
|
t.Errorf("an HTML page read as %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
|
|
body := append(header("qwen2"), make([]byte, 1<<20)...)
|
|
var rng string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
rng = r.Header.Get("Range")
|
|
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
|
|
}))
|
|
defer srv.Close()
|
|
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
|
|
if err != nil || h.Architecture != "qwen2" {
|
|
t.Fatalf("%+v, %v", h, err)
|
|
}
|
|
if !strings.HasPrefix(rng, "bytes=0-") {
|
|
t.Errorf("asked for %q rather than the start of the file", rng)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
|
|
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
|
|
t.Errorf("qwen2: %+v", v)
|
|
}
|
|
// The file that started this: downloaded, catalogued, placed, and then
|
|
// refused by every node with "unknown model architecture: 'ggmlc'".
|
|
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
|
|
t.Errorf("ggmlc: %+v", v)
|
|
}
|
|
if v := LlamaCpp("b10991", "clip"); v.Servable {
|
|
t.Errorf("a vision projector alone: %+v", v)
|
|
}
|
|
// A build llmgw has no list for is not refused, only not vouched for.
|
|
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
|
|
t.Errorf("an unknown build: %+v", v)
|
|
}
|
|
}
|
|
|
|
var timeZero = time.Time{}
|
|
|
|
// shapedHeader is a header with the keys a KV cache is sized from, written
|
|
// with the integer types llama.cpp uses for them.
|
|
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
|
|
var b bytes.Buffer
|
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
|
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
|
|
b.WriteString("GGUF")
|
|
w(uint32(3))
|
|
w(uint64(0))
|
|
w(uint64(7))
|
|
str("general.architecture")
|
|
w(uint32(tString))
|
|
str(arch)
|
|
u32(arch+".block_count", layers)
|
|
u32(arch+".attention.head_count", heads)
|
|
u32(arch+".attention.head_count_kv", kvHeads)
|
|
u32(arch+".embedding_length", embd)
|
|
u32(arch+".context_length", 32768)
|
|
str("tokenizer.ggml.tokens")
|
|
w(uint32(tArray))
|
|
w(uint32(tString))
|
|
w(uint64(1 << 20))
|
|
if !truncateTokenizer {
|
|
for i := 0; i < 1<<20; i++ {
|
|
str("t")
|
|
}
|
|
}
|
|
return b.Bytes()
|
|
}
|
|
|
|
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
|
|
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
|
|
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
|
|
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
|
|
if err != nil {
|
|
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
|
|
}
|
|
if got := h.KVBytesPerToken(); got != 196608 {
|
|
t.Errorf("%d bytes a token, expected 196608", got)
|
|
}
|
|
weights := int64(9 << 30)
|
|
want := weights + 16384*196608 + (1<<30 + weights/20)
|
|
if got := h.MemoryBytes(weights, 16384); got != want {
|
|
t.Errorf("memory %d, expected %d", got, want)
|
|
}
|
|
if (Header{}).MemoryBytes(weights, 16384) != 0 {
|
|
t.Error("a header with no shape produced a figure")
|
|
}
|
|
}
|
|
|
|
func TestArraysNestedDeepAreRefused(t *testing.T) {
|
|
var b bytes.Buffer
|
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
|
b.WriteString("GGUF")
|
|
w(uint32(3))
|
|
w(uint64(0))
|
|
w(uint64(1))
|
|
w(uint64(len("deep")))
|
|
b.WriteString("deep")
|
|
w(uint32(tArray))
|
|
for range 10_000 {
|
|
w(uint32(tArray))
|
|
w(uint64(1))
|
|
}
|
|
_, err := Read(bytes.NewReader(b.Bytes()))
|
|
if err == nil || !strings.Contains(err.Error(), "nested") {
|
|
t.Errorf("ten thousand nested arrays: %v", err)
|
|
}
|
|
}
|