An embedding model is served for embedding, with many slots
ctl model serve served one as a chat model: no --embedding, four slots, a 512-token batch, embeddings unset, and a chat smoke test it could not pass. A GGUF that pools (<arch>.pooling_type), or --embeddings, gets --embedding, 32 slots of 1024 tokens unless sized otherwise, a batch that holds a whole passage, the embeddings capability, and one vector asked for as its smoke test (canaries and updates too). An import of a hundred thousand chunks had gone at the pace of four slots with the card a third busy. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f5eb6963af
commit
efffdb66b7
8 files changed
+142
-6
No files matched your search
@@ -397,3 +397,18 @@ func TestOnlyTheLastFinishedJobsAreRemembered(t *testing.T) {
|
||||
t.Error("the oldest finished job is still remembered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmbeddingModelIsServedForEmbedding(t *testing.T) {
|
||||
// Four slots and a 512-token batch: an import went at four passages a
|
||||
// pass, and one passage of 516 tokens was refused.
|
||||
args := strings.Join(llamaArgs(Request{Embeddings: true, ContextLength: embedSlotTokens,
|
||||
CtxPool: embedSlotTokens * embedSlots}), " ")
|
||||
for _, want := range []string{"--embedding", "--ctx-size 32768", "--parallel 32", "--batch-size 1024", "--ubatch-size 1024"} {
|
||||
if !strings.Contains(args, want) {
|
||||
t.Errorf("%q not in %q", want, args)
|
||||
}
|
||||
}
|
||||
if args := strings.Join(llamaArgs(Request{ContextLength: 4096, CtxPool: 16384}), " "); strings.Contains(args, "embedding") {
|
||||
t.Errorf("a chat model got %q", args)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user