diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go index 08083e2..906a47c 100644 --- a/internal/gguf/gguf.go +++ b/internal/gguf/gguf.go @@ -25,6 +25,59 @@ type Header struct { Version uint32 Architecture string Name string + + // The shape the KV cache is sized from, from the architecture's own keys + // (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the + // file does not say, which a caller has to treat as unknown. + BlockCount uint64 + HeadCount uint64 + HeadCountKV uint64 + EmbeddingLength uint64 + // KeyLength and ValueLength are the per-head sizes when the file gives + // them; otherwise a head is EmbeddingLength / HeadCount. + KeyLength uint64 + ValueLength uint64 +} + +// KVBytesPerToken is what one token of context costs in llama.cpp's default +// f16 cache, or 0 when the header did not give the shape. +// +// Keys and values, per layer, per KV head — the KV heads rather than the +// attention heads, because with grouped-query attention a model with forty +// heads may keep only eight, and the difference is a factor of five. +func (h Header) KVBytesPerToken() int64 { + kvHeads := h.HeadCountKV + if kvHeads == 0 { + kvHeads = h.HeadCount + } + if h.BlockCount == 0 || kvHeads == 0 { + return 0 + } + k, v := h.KeyLength, h.ValueLength + if k == 0 && h.HeadCount > 0 { + k = h.EmbeddingLength / h.HeadCount + } + if v == 0 { + v = k + } + if k == 0 { + return 0 + } + const f16 = 2 + return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers +} + +// MemoryBytes is what one llama.cpp server needs for this model: the weights, +// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size, +// which --parallel divides among its slots rather than multiplies), and room +// for compute buffers and the server itself. 0 when the shape is unknown. +func (h Header) MemoryBytes(weights int64, ctxPool int) int64 { + perToken := h.KVBytesPerToken() + if perToken == 0 || weights <= 0 { + return 0 + } + overhead := int64(1<<30) + weights/20 + return weights + int64(ctxPool)*perToken + overhead } // ErrNotGGUF means the bytes are not a GGUF file. @@ -87,16 +140,42 @@ func Read(r io.Reader) (Header, error) { _, err := io.CopyN(io.Discard, br, n) return err } + // num holds the last integer value read, for the shape keys. + var num uint64 + readUint := func(size int) error { + b := make([]byte, size) + if _, err := io.ReadFull(br, b); err != nil { + return err + } + switch size { + case 1: + num = uint64(b[0]) + case 2: + num = uint64(binary.LittleEndian.Uint16(b)) + case 4: + num = uint64(binary.LittleEndian.Uint32(b)) + default: + num = binary.LittleEndian.Uint64(b) + } + return nil + } var value func(t uint32) (string, error) value = func(t uint32) (string, error) { + num = 0 switch t { - case tUint8, tInt8, tBool: + case tUint8, tInt8: + return "", readUint(1) + case tBool: return "", skip(1) case tUint16, tInt16: - return "", skip(2) - case tUint32, tInt32, tFloat32: + return "", readUint(2) + case tUint32, tInt32: + return "", readUint(4) + case tFloat32: return "", skip(4) - case tUint64, tInt64, tFloat64: + case tUint64, tInt64: + return "", readUint(8) + case tFloat64: return "", skip(8) case tString: return str() @@ -129,18 +208,27 @@ func Read(r io.Reader) (Header, error) { if err != nil { return h, err } + // Once the architecture is known, running out of the fetched bytes is + // not a failure: what the caller came for is in hand, and the shape keys + // are a bonus that a file with its tokenizer first may not reach. + partial := func(err error) (Header, error) { + if h.Architecture != "" { + return h, nil + } + return h, err + } for i := uint64(0); i < kvs; i++ { key, err := str() if err != nil { - return h, err + return partial(err) } var t uint32 if err := binary.Read(br, binary.LittleEndian, &t); err != nil { - return h, err + return partial(err) } v, err := value(t) if err != nil { - return h, err + return partial(err) } switch key { case "general.architecture": @@ -148,7 +236,25 @@ func Read(r io.Reader) (Header, error) { case "general.name": h.Name = v } - if h.Architecture != "" && h.Name != "" { + if h.Architecture != "" { + switch strings.TrimPrefix(key, h.Architecture+".") { + case "block_count": + h.BlockCount = num + case "attention.head_count": + h.HeadCount = num + case "attention.head_count_kv": + h.HeadCountKV = num + case "embedding_length": + h.EmbeddingLength = num + case "attention.key_length": + h.KeyLength = num + case "attention.value_length": + h.ValueLength = num + } + } + // The tokenizer's arrays — megabytes of vocabulary — come after the + // architecture's keys, and nothing past them is wanted. + if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 { break } } @@ -209,5 +315,7 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict { if err != nil { return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()} } - return LlamaCpp(llamaVersion, h.Architecture) + v := LlamaCpp(llamaVersion, h.Architecture) + v.Header = h + return v } diff --git a/internal/gguf/gguf_test.go b/internal/gguf/gguf_test.go index 16f853c..1303cb7 100644 --- a/internal/gguf/gguf_test.go +++ b/internal/gguf/gguf_test.go @@ -92,3 +92,54 @@ func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) { } var timeZero = time.Time{} + +// shapedHeader is a header with the keys a KV cache is sized from, written +// with the integer types llama.cpp uses for them. +func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(7)) + str("general.architecture") + w(uint32(tString)) + str(arch) + u32(arch+".block_count", layers) + u32(arch+".attention.head_count", heads) + u32(arch+".attention.head_count_kv", kvHeads) + u32(arch+".embedding_length", embd) + u32(arch+".context_length", 32768) + str("tokenizer.ggml.tokens") + w(uint32(tArray)) + w(uint32(tString)) + w(uint64(1 << 20)) + if !truncateTokenizer { + for i := 0; i < 1<<20; i++ { + str("t") + } + } + return b.Bytes() +} + +func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) { + // Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V, + // 5120 wide — 196608 bytes a token in f16, as read from the live mirror. + h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true))) + if err != nil { + t.Fatalf("a header cut off in its tokenizer failed to read: %v", err) + } + if got := h.KVBytesPerToken(); got != 196608 { + t.Errorf("%d bytes a token, expected 196608", got) + } + weights := int64(9 << 30) + want := weights + 16384*196608 + (1<<30 + weights/20) + if got := h.MemoryBytes(weights, 16384); got != want { + t.Errorf("memory %d, expected %d", got, want) + } + if (Header{}).MemoryBytes(weights, 16384) != 0 { + t.Error("a header with no shape produced a figure") + } +} diff --git a/internal/gguf/llamacpp.go b/internal/gguf/llamacpp.go index be31cf3..954ff78 100644 --- a/internal/gguf/llamacpp.go +++ b/internal/gguf/llamacpp.go @@ -180,6 +180,9 @@ type Verdict struct { // checked. Checked bool Reason string + // Header is what was read, zero when nothing was; its shape sizes the + // model's memory. + Header Header } // LlamaCpp judges an architecture against a llama.cpp build. diff --git a/internal/gguf/remote_test.go b/internal/gguf/remote_test.go new file mode 100644 index 0000000..a7f389f --- /dev/null +++ b/internal/gguf/remote_test.go @@ -0,0 +1,30 @@ +package gguf + +import ( + "os" + "strings" + "testing" +) + +// TestRealHeaders reads real files when LLMGW_GGUF_URLS names them (space +// separated), and prints the shape and memory estimate for a 16384-token pool. +// Skipped otherwise: CI has no mirror to read. +func TestRealHeaders(t *testing.T) { + urls := strings.Fields(os.Getenv("LLMGW_GGUF_URLS")) + if len(urls) == 0 { + t.Skip("LLMGW_GGUF_URLS not set") + } + for _, u := range urls { + h, err := FromURL(t.Context(), nil, u) + if err != nil { + t.Errorf("%s: %v", u, err) + continue + } + t.Logf("%s: arch=%s layers=%d heads=%d kv_heads=%d embd=%d key_len=%d kv/token=%d B, kv@16k=%.2f GiB", + u[strings.LastIndex(u, "/")+1:], h.Architecture, h.BlockCount, h.HeadCount, h.HeadCountKV, + h.EmbeddingLength, h.KeyLength, h.KVBytesPerToken(), float64(16384*h.KVBytesPerToken())/(1<<30)) + if h.KVBytesPerToken() == 0 { + t.Errorf("%s: no KV shape read", u) + } + } +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 2ff667f..98cfa1e 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -49,6 +49,11 @@ type serveForm struct { // 3.5 GB for a KV pool. Nothing was wrong with placement — it was never // told what the model needed. MinMemoryGB int + // MemoryBasis says how MinMemoryGB was arrived at, for the form to show. + MemoryBasis string + // CtxPool is the engine context pool the form starts with, and the one + // the memory figure assumes. + CtxPool int // Refusal is set when this version cannot be served as it stands, and says // why. Offered and then failing at placement is worse than not offered. @@ -63,6 +68,7 @@ type serveForm struct { func planServe(st mirror.Status, files []mirror.Status, base string, fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict, ) serveForm { + var header gguf.Header f := serveForm{ Model: st.Model, Version: st.Version, @@ -101,10 +107,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string, case "llamacpp": f.Profile = "cpu" if check != nil { - if v := check(f.Source); !v.Servable { + v := check(f.Source) + if !v.Servable { f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason return f } + header = v.Header } default: f.Profile = "gpu" @@ -123,7 +131,27 @@ func planServe(st mirror.Status, files []mirror.Status, base string, "same model runs on the CPU nodes" } } - f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone) + f.CtxPool = defaultCtxPool + f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), "weights plus half again plus two" + weights := st.BytesTotal + if weights <= 0 { + weights = st.BytesDone + } + if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 { + kvHeads := header.HeadCountKV + if kvHeads == 0 { + kvHeads = header.HeadCount + } + // From the model's own shape: its KV cache for the pool the form + // starts with, which ranges from under half a gigabyte to over five + // across the models on the live mirror at the same 16k — a spread the + // flat "half again" could not see. + f.MinMemoryGB = int((b + 1<<30 - 1) >> 30) + f.MemoryBasis = fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens (%d layers × %d KV heads) + %.1f GB for the engine", + float64(weights)/(1<<30), float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30), + defaultCtxPool, header.BlockCount, kvHeads, + float64(int64(1<<30)+weights/20)/(1<<30)) + } if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest { f.Warning = strings.TrimSuffix(f.Warning, ".") + fmt.Sprintf("%sthe largest node has %d GB and this needs about %d", @@ -353,3 +381,7 @@ func modeOf(sc catalog.Scaling) string { return modeStopped } } + +// defaultCtxPool is the llama.cpp context pool the serve form starts with. +// --parallel divides it among slots; it does not multiply it. +const defaultCtxPool = 16384 diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 4f7dce7..64477ca 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -147,11 +147,11 @@ reachable from this machine.

+
-