The serve form asked for weights plus half again plus two gigabytes. That ignores the one thing that varies most: the KV cache. At the same 16k pool, the models on the live mirror need from 0.44 GB of cache (Qwen2.5-1.5B, two KV heads) to 5.25 GB (Gemma-2-9B, 256-wide heads) — so the flat guess overcounted the small models, which on 7 GB nodes means refusing placements that fit, and undercounted the ones with wide or many KV heads, which means an engine killed for memory. The GGUF header already carries the shape. The reader now also takes each architecture's block_count, head counts, embedding length and key/value lengths, reading past general.* until the tokenizer — and a header whose tokenizer comes first and runs past the fetched bytes still returns what it has rather than failing. The estimate is weights, plus the f16 KV cache for the context pool (llama.cpp's --ctx-size, which --parallel divides rather than multiplies), plus a gigabyte and five percent for the engine. It reuses the header the servability check already fetched, and the form says what it is made of. Checked against all twelve GGUF models on the live mirror, whose shapes read back as their published architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
146 lines
4.3 KiB
Go
146 lines
4.3 KiB
Go
package gguf
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// header builds a GGUF v3 header: an array before the architecture, the way
|
|
// real files put tokenizer data, so skipping values is exercised.
|
|
func header(arch string) []byte {
|
|
var b bytes.Buffer
|
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
|
b.WriteString("GGUF")
|
|
w(uint32(3))
|
|
w(uint64(0)) // tensors
|
|
w(uint64(4)) // kvs
|
|
str("general.alignment")
|
|
w(uint32(tUint32))
|
|
w(uint32(32))
|
|
str("some.list")
|
|
w(uint32(tArray))
|
|
w(uint32(tString))
|
|
w(uint64(2))
|
|
str("a")
|
|
str("bc")
|
|
str("general.architecture")
|
|
w(uint32(tString))
|
|
str(arch)
|
|
str("general.name")
|
|
w(uint32(tString))
|
|
str("A Model")
|
|
return b.Bytes()
|
|
}
|
|
|
|
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
|
|
h, err := Read(bytes.NewReader(header("ggmlc")))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
|
|
t.Errorf("header = %+v", h)
|
|
}
|
|
}
|
|
|
|
func TestSomethingElseIsNotAGGUF(t *testing.T) {
|
|
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
|
|
t.Errorf("an HTML page read as %v", err)
|
|
}
|
|
}
|
|
|
|
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
|
|
body := append(header("qwen2"), make([]byte, 1<<20)...)
|
|
var rng string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
rng = r.Header.Get("Range")
|
|
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
|
|
}))
|
|
defer srv.Close()
|
|
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
|
|
if err != nil || h.Architecture != "qwen2" {
|
|
t.Fatalf("%+v, %v", h, err)
|
|
}
|
|
if !strings.HasPrefix(rng, "bytes=0-") {
|
|
t.Errorf("asked for %q rather than the start of the file", rng)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
|
|
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
|
|
t.Errorf("qwen2: %+v", v)
|
|
}
|
|
// The file that started this: downloaded, catalogued, placed, and then
|
|
// refused by every node with "unknown model architecture: 'ggmlc'".
|
|
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
|
|
t.Errorf("ggmlc: %+v", v)
|
|
}
|
|
if v := LlamaCpp("b10991", "clip"); v.Servable {
|
|
t.Errorf("a vision projector alone: %+v", v)
|
|
}
|
|
// A build llmgw has no list for is not refused, only not vouched for.
|
|
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
|
|
t.Errorf("an unknown build: %+v", v)
|
|
}
|
|
}
|
|
|
|
var timeZero = time.Time{}
|
|
|
|
// shapedHeader is a header with the keys a KV cache is sized from, written
|
|
// with the integer types llama.cpp uses for them.
|
|
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
|
|
var b bytes.Buffer
|
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
|
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
|
|
b.WriteString("GGUF")
|
|
w(uint32(3))
|
|
w(uint64(0))
|
|
w(uint64(7))
|
|
str("general.architecture")
|
|
w(uint32(tString))
|
|
str(arch)
|
|
u32(arch+".block_count", layers)
|
|
u32(arch+".attention.head_count", heads)
|
|
u32(arch+".attention.head_count_kv", kvHeads)
|
|
u32(arch+".embedding_length", embd)
|
|
u32(arch+".context_length", 32768)
|
|
str("tokenizer.ggml.tokens")
|
|
w(uint32(tArray))
|
|
w(uint32(tString))
|
|
w(uint64(1 << 20))
|
|
if !truncateTokenizer {
|
|
for i := 0; i < 1<<20; i++ {
|
|
str("t")
|
|
}
|
|
}
|
|
return b.Bytes()
|
|
}
|
|
|
|
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
|
|
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
|
|
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
|
|
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
|
|
if err != nil {
|
|
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
|
|
}
|
|
if got := h.KVBytesPerToken(); got != 196608 {
|
|
t.Errorf("%d bytes a token, expected 196608", got)
|
|
}
|
|
weights := int64(9 << 30)
|
|
want := weights + 16384*196608 + (1<<30 + weights/20)
|
|
if got := h.MemoryBytes(weights, 16384); got != want {
|
|
t.Errorf("memory %d, expected %d", got, want)
|
|
}
|
|
if (Header{}).MemoryBytes(weights, 16384) != 0 {
|
|
t.Error("a header with no shape produced a figure")
|
|
}
|
|
}
|