An embedding model is served for embedding, with many slots

ctl model serve served one as a chat model: no --embedding, four slots, a
512-token batch, embeddings unset, and a chat smoke test it could not
pass. A GGUF that pools (<arch>.pooling_type), or --embeddings, gets
--embedding, 32 slots of 1024 tokens unless sized otherwise, a batch that
holds a whole passage, the embeddings capability, and one vector asked for
as its smoke test (canaries and updates too). An import of a hundred
thousand chunks had gone at the pace of four slots with the card a third
busy.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-14 09:19:57 +02:00
1 parent f5eb6963af
commit efffdb66b7
8 files changed
+142 -6

No files matched your search

+1 -1
View File
@@ -92,7 +92,7 @@ func (s *Server) addCandidate(ctx context.Context, j *Job, p prepared, cn catalo
s.step(j, "smoke test", Running, "asking "+r.ID+" to say hello")
var reply string
var tps float64
if reply, tps, err = s.smoke(ctx, r, cand); err == nil {
if reply, tps, err = s.smoke(ctx, r, cand, s.embeds(cand)); err == nil {
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
} else {
s.step(j, "smoke test", Failed, err.Error())
+86 -4
View File
@@ -153,6 +153,13 @@ type Request struct {
// is written into the catalog, where clients and editors read it. Check
// a running replica with llmgw ctl tool-check before saying so.
Tools bool `json:"tools,omitempty"`
// Embeddings serves it as an embedding model, which its GGUF header
// says by itself for most: --embedding, many slots, a batch that holds
// a whole passage, /v1/embeddings.
Embeddings bool `json:"embeddings,omitempty"`
// sized is the caller having said --ctx-pool or --context-length, which
// an embedding model's defaults then leave alone.
sized bool
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
@@ -268,6 +275,7 @@ func (s *Server) Start(req Request) (Job, error) {
if req.GuardFormat != "" && req.Replicas == 0 {
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
}
req.sized = req.CtxPool > 0 || req.ContextLength > 0
if req.CtxPool <= 0 {
req.CtxPool = 16384
}
@@ -473,6 +481,16 @@ func (s *Server) run(j *Job) {
}
header = v.Header
}
if header.Embedding {
req.Embeddings = true
}
if req.Embeddings && !req.sized {
// Many passages at once rather than four: an import of a hundred
// thousand chunks went at the pace of four slots with the card a
// third busy.
req.ContextLength, req.CtxPool = embedSlotTokens, embedSlotTokens*embedSlots
}
j.Request = req
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
gb(set.Bytes), gb(int64(req.CtxPool)*header.KVBytesPerToken()), req.CtxPool)
@@ -587,10 +605,24 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
// slots of 8192, and the gateway, told 32768, sent requests the engine then
// refused.
func llamaArgs(req Request) []string {
return []string{"--ctx-size", strconv.Itoa(req.CtxPool),
args := []string{"--ctx-size", strconv.Itoa(req.CtxPool),
"--parallel", strconv.Itoa(max(1, req.CtxPool/req.ContextLength))}
if req.Embeddings {
// A passage is embedded in one batch: with the default 512, one of
// 516 tokens was refused.
n := strconv.Itoa(req.ContextLength)
args = append([]string{"--embedding"}, append(args, "--batch-size", n, "--ubatch-size", n)...)
}
return args
}
// An embedding model's slots, when its caller did not size them: passages
// of up to a thousand tokens, thirty-two at a time.
const (
embedSlotTokens = 1024
embedSlots = 32
)
// maxOutput is the longest answer the catalog allows: a quarter of the
// context, from 1024 to 8192. 1024 everywhere cut an editor's agent off in
// the middle of the file it was writing.
@@ -620,7 +652,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: maxOutput(req.ContextLength),
Profiles: profs,
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools},
Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools, Embeddings: req.Embeddings},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
}
if req.GuardFormat != "" {
@@ -655,7 +687,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
// 6. Does it answer.
s.step(j, "smoke test", Running, "asking "+rep.ID+" to say hello")
reply, tps, err := s.smoke(ctx, rep, name)
reply, tps, err := s.smoke(ctx, rep, name, req.Embeddings)
if err != nil {
s.fail(j, "smoke test", err)
return
@@ -859,7 +891,10 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
}
// smoke sends one short request straight to the replica, with its engine key.
func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) {
func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string, embeddings bool) (string, float64, error) {
if embeddings {
return s.smokeEmbeddings(ctx, r, name)
}
body, _ := json.Marshal(map[string]any{
"model": name,
"max_tokens": 32,
@@ -989,3 +1024,50 @@ func pickSafetensors(repo hf.Repo, _ Request) (hf.FileSet, error) {
}
return set, nil
}
// smokeEmbeddings asks an embedding model for one vector, and says its
// length: an embedding model does not chat.
func (s *Server) smokeEmbeddings(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) {
body, _ := json.Marshal(map[string]any{"model": name, "input": "hello"})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
strings.TrimRight(r.Endpoint, "/")+"/v1/embeddings", bytes.NewReader(body))
if err != nil {
return "", 0, err
}
req.Header.Set("Content-Type", "application/json")
if k := s.deps.Fleet.EngineKey(r.ID); k != "" {
req.Header.Set("Authorization", "Bearer "+k)
}
resp, err := s.deps.HTTP.Do(req)
if err != nil {
return "", 0, err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode != http.StatusOK {
return "", 0, fmt.Errorf("the replica answered %s: %s", resp.Status, strings.TrimSpace(string(b)))
}
var out struct {
Data []struct {
Embedding []float64 `json:"embedding"`
} `json:"data"`
}
if err := json.Unmarshal(b, &out); err != nil || len(out.Data) == 0 || len(out.Data[0].Embedding) == 0 {
return "", 0, fmt.Errorf("the replica answered no vector: %s", strings.TrimSpace(string(b)))
}
return fmt.Sprintf("a vector of %d dimensions", len(out.Data[0].Embedding)), 0, nil
}
// embeds is whether the catalog serves name as an embedding model, for a
// smoke test of one already there (a canary, an update).
func (s *Server) embeds(name string) bool {
if s.deps.Current == nil {
return false
}
for _, m := range s.deps.Current() {
if m.Name == name {
return m.Capabilities.Embeddings
}
}
return false
}
@@ -397,3 +397,18 @@ func TestOnlyTheLastFinishedJobsAreRemembered(t *testing.T) {
t.Error("the oldest finished job is still remembered")
}
}
func TestAnEmbeddingModelIsServedForEmbedding(t *testing.T) {
// Four slots and a 512-token batch: an import went at four passages a
// pass, and one passage of 516 tokens was refused.
args := strings.Join(llamaArgs(Request{Embeddings: true, ContextLength: embedSlotTokens,
CtxPool: embedSlotTokens * embedSlots}), " ")
for _, want := range []string{"--embedding", "--ctx-size 32768", "--parallel 32", "--batch-size 1024", "--ubatch-size 1024"} {
if !strings.Contains(args, want) {
t.Errorf("%q not in %q", want, args)
}
}
if args := strings.Join(llamaArgs(Request{ContextLength: 4096, CtxPool: 16384}), " "); strings.Contains(args, "embedding") {
t.Errorf("a chat model got %q", args)
}
}
+1 -1
View File
@@ -689,7 +689,7 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
}
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)
reply, tps, err := s.smoke(ctx, ready[0], name, s.embeds(name))
if err != nil {
s.step(j, "smoke test", Failed, err.Error())
return err