An embedding model is served for embedding, with many slots
ctl model serve served one as a chat model: no --embedding, four slots, a 512-token batch, embeddings unset, and a chat smoke test it could not pass. A GGUF that pools (<arch>.pooling_type), or --embeddings, gets --embedding, 32 slots of 1024 tokens unless sized otherwise, a batch that holds a whole passage, the embeddings capability, and one vector asked for as its smoke test (canaries and updates too). An import of a hundred thousand chunks had gone at the pace of four slots with the card a third busy. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f5eb6963af
commit
efffdb66b7
8 files changed
+142
-6
No files matched your search
@@ -41,6 +41,10 @@ type Header struct {
|
||||
// them; otherwise a head is EmbeddingLength / HeadCount.
|
||||
KeyLength uint64
|
||||
ValueLength uint64
|
||||
// Embedding is whether the model pools its tokens into one vector
|
||||
// (<arch>.pooling_type set and not none): an embedding model, which
|
||||
// llama.cpp serves with --embedding and does not chat.
|
||||
Embedding bool
|
||||
}
|
||||
|
||||
// KVBytesPerToken is what one token of context costs in llama.cpp's default
|
||||
@@ -265,6 +269,8 @@ func Read(r io.Reader) (Header, error) {
|
||||
h.KeyLength = num
|
||||
case "attention.value_length":
|
||||
h.ValueLength = num
|
||||
case "pooling_type":
|
||||
h.Embedding = num > 0
|
||||
}
|
||||
}
|
||||
// The tokenizer's arrays — megabytes of vocabulary — come after the
|
||||
|
||||
@@ -163,3 +163,28 @@ func TestArraysNestedDeepAreRefused(t *testing.T) {
|
||||
t.Errorf("ten thousand nested arrays: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmbeddingModelIsKnownByItsPooling(t *testing.T) {
|
||||
build := func(pooling uint32) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(3))
|
||||
str("general.architecture")
|
||||
w(uint32(tString))
|
||||
str("bert")
|
||||
u32("bert.block_count", 24)
|
||||
u32("bert.pooling_type", pooling)
|
||||
return b.Bytes()
|
||||
}
|
||||
if h, err := Read(bytes.NewReader(build(2))); err != nil || !h.Embedding {
|
||||
t.Errorf("CLS pooling: %+v %v", h, err)
|
||||
}
|
||||
if h, err := Read(bytes.NewReader(build(0))); err != nil || h.Embedding {
|
||||
t.Errorf("no pooling: %+v %v", h, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user