serve: tool calls declared, slots that hold the context, longer answers
CI / test (push) Failing after 2m41s

A --context-length 32768 --ctx-pool 32768 model got llama.cpp's default four
slots of 8192, and refused what the gateway admitted; --parallel is now
what the pool holds of the context. max_output was 1024 for every model,
which cut an editor's agent off mid-file; it is now a quarter of the
context, from 1024 to 8192. --tools writes capabilities.tools, which the
Continue configuration now reads, as it reads max_output.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 09:30:40 +02:00
1 parent df6d243ee7
commit 481523ca46
8 files changed
+111 -8

No files matched your search

+31 -5
View File
@@ -140,9 +140,14 @@ type Request struct {
// llamaguard3, shieldgemma or json. Empty serves clients.
GuardFormat string `json:"guard_format,omitempty"`
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
// use. They default to 16384 and 4096, what four parallel slots divide.
// use. They default to 16384 and 4096, what four parallel slots divide;
// llama.cpp is given as many slots as the pool holds requests.
CtxPool int `json:"ctx_pool,omitempty"`
ContextLength int `json:"context_length,omitempty"`
// Tools says the model makes tool calls the engine returns as calls: it
// is written into the catalog, where clients and editors read it. Check
// a running replica with llmgw ctl tool-check before saying so.
Tools bool `json:"tools,omitempty"`
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
@@ -261,6 +266,10 @@ func (s *Server) Start(req Request) (Job, error) {
if req.ContextLength <= 0 {
req.ContextLength = 4096
}
if req.ContextLength > req.CtxPool {
return Job{}, fmt.Errorf("a request of %d tokens cannot fit a pool of %d: give --ctx-pool at least --context-length",
req.ContextLength, req.CtxPool)
}
s.mu.Lock()
s.seq++
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
@@ -545,12 +554,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
switch {
case o.CPU:
p.Name = "cpu"
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.Args = llamaArgs(req)
p.MinMemoryGB, p.CPUCores = memGB, cores
case o.Engine == "llamacpp":
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.Args = llamaArgs(req)
default:
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
@@ -564,6 +573,23 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
return out
}
// llamaArgs size llama.cpp's pool and its slots together. The pool is shared
// by the slots, so slots of the context the catalog promises are as many as
// the pool holds: the engine's default of four turned a 32768 pool into
// slots of 8192, and the gateway, told 32768, sent requests the engine then
// refused.
func llamaArgs(req Request) []string {
return []string{"--ctx-size", strconv.Itoa(req.CtxPool),
"--parallel", strconv.Itoa(max(1, req.CtxPool/req.ContextLength))}
}
// maxOutput is the longest answer the catalog allows: a quarter of the
// context, from 1024 to 8192. 1024 everywhere cut an editor's agent off in
// the middle of the file it was writing.
func maxOutput(contextLength int) int {
return min(8192, max(1024, contextLength/4))
}
// addAndStart puts the model in the catalog with its profile, waits for a
// replica and asks it to answer.
// hardwareExists is whether the cluster has what the ways chosen need, busy
@@ -583,10 +609,10 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: maxOutput(req.ContextLength),
Profiles: profs,
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
}
if req.GuardFormat != "" {
+31
View File
@@ -0,0 +1,31 @@
package modelserve
import (
"slices"
"testing"
)
func TestLlamaCppIsGivenTheSlotsTheCatalogPromises(t *testing.T) {
// A 32768 pool for 32768-token requests is one slot: the engine's own
// four made slots of 8192, and requests the gateway admitted were refused.
for _, c := range []struct {
pool, ctx int
want []string
}{
{32768, 32768, []string{"--ctx-size", "32768", "--parallel", "1"}},
{16384, 4096, []string{"--ctx-size", "16384", "--parallel", "4"}},
{65536, 16384, []string{"--ctx-size", "65536", "--parallel", "4"}},
} {
if got := llamaArgs(Request{CtxPool: c.pool, ContextLength: c.ctx}); !slices.Equal(got, c.want) {
t.Errorf("pool %d, context %d: %v", c.pool, c.ctx, got)
}
}
}
func TestTheLongestAnswerGrowsWithTheContext(t *testing.T) {
for ctx, want := range map[int]int{2048: 1024, 4096: 1024, 16384: 4096, 32768: 8192, 131072: 8192} {
if got := maxOutput(ctx); got != want {
t.Errorf("context %d: max output %d, want %d", ctx, got, want)
}
}
}