An embedding model is served for embedding, with many slots
ctl model serve served one as a chat model: no --embedding, four slots, a 512-token batch, embeddings unset, and a chat smoke test it could not pass. A GGUF that pools (<arch>.pooling_type), or --embeddings, gets --embedding, 32 slots of 1024 tokens unless sized otherwise, a batch that holds a whole passage, the embeddings capability, and one vector asked for as its smoke test (canaries and updates too). An import of a hundred thousand chunks had gone at the pace of four slots with the card a third busy. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f5eb6963af
commit
efffdb66b7
8 files changed
+142
-6
No files matched your search
@@ -92,7 +92,7 @@ func (s *Server) addCandidate(ctx context.Context, j *Job, p prepared, cn catalo
|
||||
s.step(j, "smoke test", Running, "asking "+r.ID+" to say hello")
|
||||
var reply string
|
||||
var tps float64
|
||||
if reply, tps, err = s.smoke(ctx, r, cand); err == nil {
|
||||
if reply, tps, err = s.smoke(ctx, r, cand, s.embeds(cand)); err == nil {
|
||||
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
|
||||
} else {
|
||||
s.step(j, "smoke test", Failed, err.Error())
|
||||
|
||||
Reference in new issue
Block a user