An embedding model is served for embedding, with many slots

ctl model serve served one as a chat model: no --embedding, four slots, a
512-token batch, embeddings unset, and a chat smoke test it could not
pass. A GGUF that pools (<arch>.pooling_type), or --embeddings, gets
--embedding, 32 slots of 1024 tokens unless sized otherwise, a batch that
holds a whole passage, the embeddings capability, and one vector asked for
as its smoke test (canaries and updates too). An import of a hundred
thousand chunks had gone at the pace of four slots with the card a third
busy.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-14 09:19:57 +02:00
1 parent f5eb6963af
commit efffdb66b7
8 files changed
+142 -6

No files matched your search

+6
View File
@@ -41,6 +41,10 @@ type Header struct {
// them; otherwise a head is EmbeddingLength / HeadCount.
KeyLength uint64
ValueLength uint64
// Embedding is whether the model pools its tokens into one vector
// (<arch>.pooling_type set and not none): an embedding model, which
// llama.cpp serves with --embedding and does not chat.
Embedding bool
}
// KVBytesPerToken is what one token of context costs in llama.cpp's default
@@ -265,6 +269,8 @@ func Read(r io.Reader) (Header, error) {
h.KeyLength = num
case "attention.value_length":
h.ValueLength = num
case "pooling_type":
h.Embedding = num > 0
}
}
// The tokenizer's arrays — megabytes of vocabulary — come after the
+25
View File
@@ -163,3 +163,28 @@ func TestArraysNestedDeepAreRefused(t *testing.T) {
t.Errorf("ten thousand nested arrays: %v", err)
}
}
func TestAnEmbeddingModelIsKnownByItsPooling(t *testing.T) {
build := func(pooling uint32) []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(3))
str("general.architecture")
w(uint32(tString))
str("bert")
u32("bert.block_count", 24)
u32("bert.pooling_type", pooling)
return b.Bytes()
}
if h, err := Read(bytes.NewReader(build(2))); err != nil || !h.Embedding {
t.Errorf("CLS pooling: %+v %v", h, err)
}
if h, err := Read(bytes.NewReader(build(0))); err != nil || h.Embedding {
t.Errorf("no pooling: %+v %v", h, err)
}
}