Files
llmgw/internal/gguf/gguf_test.go
T
Ilker ManapandClaude Opus 5.5 c22528c26e
CI / test (push) Failing after 30m3s
A model that does not fit its card is not added to fail; weights in use are not removed
Apertus 70B with a 65536-token pool, and then 8B with 131072, on a 24 GB
card loaded their weights and failed to make their context on every start,
the 8B taking the card from the 70B each time only to fail. ctl model serve
now shrinks a pool the card cannot hold beside the weights (KV cache from
the GGUF's shape, 90% of the card, a gigabyte for the engine), saying so,
and refuses weights that do not fit at all; the portal's serve form refuses
such a pool and says the largest that fits.

The mirror page removed the 70B's weights while the catalog still had the
model, whose replicas went on loading the node's copy with nothing said.
Removing weights a catalog model loads is refused, naming the model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-15 00:22:01 +02:00

213 lines
6.2 KiB
Go

package gguf
import (
"bytes"
"context"
"encoding/binary"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// header builds a GGUF v3 header: an array before the architecture, the way
// real files put tokenizer data, so skipping values is exercised.
func header(arch string) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0)) // tensors
w(uint64(4)) // kvs
str("general.alignment")
w(uint32(tUint32))
w(uint32(32))
str("some.list")
w(uint32(tArray))
w(uint32(tString))
w(uint64(2))
str("a")
str("bc")
str("general.architecture")
w(uint32(tString))
str(arch)
str("general.name")
w(uint32(tString))
str("A Model")
return b.Bytes()
}
func TestTheArchitectureIsReadFromTheHeader(t *testing.T) {
h, err := Read(bytes.NewReader(header("ggmlc")))
if err != nil {
t.Fatal(err)
}
if h.Architecture != "ggmlc" || h.Name != "A Model" || h.Version != 3 {
t.Errorf("header = %+v", h)
}
}
func TestSomethingElseIsNotAGGUF(t *testing.T) {
if _, err := Read(strings.NewReader("<html>not found</html>")); !errors.Is(err, ErrNotGGUF) {
t.Errorf("an HTML page read as %v", err)
}
}
func TestOnlyTheStartOfARemoteFileIsFetched(t *testing.T) {
body := append(header("qwen2"), make([]byte, 1<<20)...)
var rng string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
rng = r.Header.Get("Range")
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(body))
}))
defer srv.Close()
h, err := FromURL(context.Background(), nil, srv.URL+"/m.gguf")
if err != nil || h.Architecture != "qwen2" {
t.Fatalf("%+v, %v", h, err)
}
if !strings.HasPrefix(rng, "bytes=0-") {
t.Errorf("asked for %q rather than the start of the file", rng)
}
}
func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
if v := LlamaCpp("b10991", "qwen2"); !v.Servable || !v.Checked {
t.Errorf("qwen2: %+v", v)
}
// The file that started this: downloaded, catalogued, placed, and then
// refused by every node with "unknown model architecture: 'ggmlc'".
if v := LlamaCpp("b10991", "ggmlc"); v.Servable || !strings.Contains(v.Reason, "cannot load") {
t.Errorf("ggmlc: %+v", v)
}
if v := LlamaCpp("b10991", "clip"); v.Servable {
t.Errorf("a vision projector alone: %+v", v)
}
// A build llmgw has no list for is not refused, only not vouched for.
if v := LlamaCpp("b99999", "ggmlc"); !v.Servable || v.Checked {
t.Errorf("an unknown build: %+v", v)
}
}
var timeZero = time.Time{}
// shapedHeader is a header with the keys a KV cache is sized from, written
// with the integer types llama.cpp uses for them.
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(7))
str("general.architecture")
w(uint32(tString))
str(arch)
u32(arch+".block_count", layers)
u32(arch+".attention.head_count", heads)
u32(arch+".attention.head_count_kv", kvHeads)
u32(arch+".embedding_length", embd)
u32(arch+".context_length", 32768)
str("tokenizer.ggml.tokens")
w(uint32(tArray))
w(uint32(tString))
w(uint64(1 << 20))
if !truncateTokenizer {
for i := 0; i < 1<<20; i++ {
str("t")
}
}
return b.Bytes()
}
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
if err != nil {
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
}
if got := h.KVBytesPerToken(); got != 196608 {
t.Errorf("%d bytes a token, expected 196608", got)
}
weights := int64(9 << 30)
want := weights + 16384*196608 + (1<<30 + weights/20)
if got := h.MemoryBytes(weights, 16384); got != want {
t.Errorf("memory %d, expected %d", got, want)
}
if (Header{}).MemoryBytes(weights, 16384) != 0 {
t.Error("a header with no shape produced a figure")
}
}
func TestArraysNestedDeepAreRefused(t *testing.T) {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(1))
w(uint64(len("deep")))
b.WriteString("deep")
w(uint32(tArray))
for range 10_000 {
w(uint32(tArray))
w(uint64(1))
}
_, err := Read(bytes.NewReader(b.Bytes()))
if err == nil || !strings.Contains(err.Error(), "nested") {
t.Errorf("ten thousand nested arrays: %v", err)
}
}
func TestAnEmbeddingModelIsKnownByItsPooling(t *testing.T) {
build := func(pooling uint32) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(3))
str("general.architecture")
w(uint32(tString))
str("bert")
u32("bert.block_count", 24)
u32("bert.pooling_type", pooling)
return b.Bytes()
}
if h, err := Read(bytes.NewReader(build(2))); err != nil || !h.Embedding {
t.Errorf("CLS pooling: %+v %v", h, err)
}
if h, err := Read(bytes.NewReader(build(0))); err != nil || h.Embedding {
t.Errorf("no pooling: %+v %v", h, err)
}
}
func TestAPoolIsSizedToTheCard(t *testing.T) {
// Apertus-8B's shape: 32 layers, 8 KV heads of 128, 128 KiB a token.
h, err := Read(bytes.NewReader(shapedHeader("apertus", 32, 32, 8, 4096, true)))
if err != nil {
t.Fatal(err)
}
weights := int64(8.5 * (1 << 30))
most := h.PoolThatFits(weights, ClassMemGB("6000-24g"), 1)
if most < 90000 || most > 105000 {
t.Errorf("an 8B on 24 GB holds %d tokens of pool", most)
}
if most >= 131072 {
t.Error("the 131072 pool that failed on the card would pass")
}
if got := h.PoolThatFits(int64(30*(1<<30)), 24, 1); got != -1 {
t.Errorf("30 GB of weights on 24 GB: %d", got)
}
if ClassMemGB("t4-15g") != 15 || ClassMemGB("cpu") != 0 || ClassMemGB("v100-32g") != 32 {
t.Error("class memory misread")
}
}