serve: tool calls declared, slots that hold the context, longer answers
CI / test (push) Failing after 2m41s

A --context-length 32768 --ctx-pool 32768 model got llama.cpp's default four
slots of 8192, and refused what the gateway admitted; --parallel is now
what the pool holds of the context. max_output was 1024 for every model,
which cut an editor's agent off mid-file; it is now a quarter of the
context, from 1024 to 8192. --tools writes capabilities.tools, which the
Continue configuration now reads, as it reads max_output.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 09:30:40 +02:00
1 parent df6d243ee7
commit 481523ca46
8 files changed
+111 -8

No files matched your search

+14
View File
@@ -53,8 +53,22 @@ Two control planes, run in anger on the reference cluster.
dashboard as well as the API; a proxy left pointing at a standby showed an
empty cluster. The leader trusts the control planes' `X-Forwarded-For`.
- `ctl model serve --tools` says in the catalog that a model makes tool
calls, for agents and editors.
### Fixed
- **Continue's agent did nothing.** The configuration the portal writes told
Continue every model uses tools, and nothing of its context: a model whose
calls came back as text stalled the agent, and prompts over a CPU
replica's 4096 tokens were refused. It now says `tool_use` only for a
model whose catalog says tools, and the context and longest answer the
gateway allows.
- `ctl model serve` gave llama.cpp four slots whatever was asked: a
`--context-length 32768 --ctx-pool 32768` model had slots of 8192 and
refused what the gateway admitted. Slots are now what the pool holds of
the context. The longest answer grows with the context (a quarter, from
1024 to 8192) rather than being 1024 for every model.
- `ctl upgrade` upgrades the leader last wherever it is run; run on the
standby, it upgraded the leader first and the leadership moved twice.
- **Failover took 40 seconds; it takes a few.** A stopping control plane held
+9
View File
@@ -184,6 +184,15 @@ asks for, send the results back. Whether a model calls tools well is the
model's business, not the gateway's; ask your operator which of the served
models are set up for it.
**Editors' agents need such a model.** Continue's Agent mode, Cline and
Codex act only through tool calls: with a model whose calls come back as
text — `<tool_call>{...}` or a JSON block in the answer, `tool_calls` empty —
they chat but never read a file or run a command. Qwen2.5-Coder is one;
Qwen3-Coder is not. The configuration the portal writes for Continue says
`tool_use` only for a model whose catalog entry says tools, and gives
Continue the model's context and longest answer, so it never sends a request
the gateway refuses.
```python
import json
+2
View File
@@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
f.BoolVar(&req.Tools, "tools", false,
"say it makes tool calls (for agents and editors); check a running one with llmgw ctl tool-check")
f.StringVar(&req.GuardFormat, "guard-format", "",
"serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients")
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
+31 -5
View File
@@ -140,9 +140,14 @@ type Request struct {
// llamaguard3, shieldgemma or json. Empty serves clients.
GuardFormat string `json:"guard_format,omitempty"`
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
// use. They default to 16384 and 4096, what four parallel slots divide.
// use. They default to 16384 and 4096, what four parallel slots divide;
// llama.cpp is given as many slots as the pool holds requests.
CtxPool int `json:"ctx_pool,omitempty"`
ContextLength int `json:"context_length,omitempty"`
// Tools says the model makes tool calls the engine returns as calls: it
// is written into the catalog, where clients and editors read it. Check
// a running replica with llmgw ctl tool-check before saying so.
Tools bool `json:"tools,omitempty"`
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
@@ -261,6 +266,10 @@ func (s *Server) Start(req Request) (Job, error) {
if req.ContextLength <= 0 {
req.ContextLength = 4096
}
if req.ContextLength > req.CtxPool {
return Job{}, fmt.Errorf("a request of %d tokens cannot fit a pool of %d: give --ctx-pool at least --context-length",
req.ContextLength, req.CtxPool)
}
s.mu.Lock()
s.seq++
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
@@ -545,12 +554,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
switch {
case o.CPU:
p.Name = "cpu"
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.Args = llamaArgs(req)
p.MinMemoryGB, p.CPUCores = memGB, cores
case o.Engine == "llamacpp":
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.Args = llamaArgs(req)
default:
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
@@ -564,6 +573,23 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
return out
}
// llamaArgs size llama.cpp's pool and its slots together. The pool is shared
// by the slots, so slots of the context the catalog promises are as many as
// the pool holds: the engine's default of four turned a 32768 pool into
// slots of 8192, and the gateway, told 32768, sent requests the engine then
// refused.
func llamaArgs(req Request) []string {
return []string{"--ctx-size", strconv.Itoa(req.CtxPool),
"--parallel", strconv.Itoa(max(1, req.CtxPool/req.ContextLength))}
}
// maxOutput is the longest answer the catalog allows: a quarter of the
// context, from 1024 to 8192. 1024 everywhere cut an editor's agent off in
// the middle of the file it was writing.
func maxOutput(contextLength int) int {
return min(8192, max(1024, contextLength/4))
}
// addAndStart puts the model in the catalog with its profile, waits for a
// replica and asks it to answer.
// hardwareExists is whether the cluster has what the ways chosen need, busy
@@ -583,10 +609,10 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: maxOutput(req.ContextLength),
Profiles: profs,
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
}
if req.GuardFormat != "" {
+31
View File
@@ -0,0 +1,31 @@
package modelserve
import (
"slices"
"testing"
)
func TestLlamaCppIsGivenTheSlotsTheCatalogPromises(t *testing.T) {
// A 32768 pool for 32768-token requests is one slot: the engine's own
// four made slots of 8192, and requests the gateway admitted were refused.
for _, c := range []struct {
pool, ctx int
want []string
}{
{32768, 32768, []string{"--ctx-size", "32768", "--parallel", "1"}},
{16384, 4096, []string{"--ctx-size", "16384", "--parallel", "4"}},
{65536, 16384, []string{"--ctx-size", "65536", "--parallel", "4"}},
} {
if got := llamaArgs(Request{CtxPool: c.pool, ContextLength: c.ctx}); !slices.Equal(got, c.want) {
t.Errorf("pool %d, context %d: %v", c.pool, c.ctx, got)
}
}
}
func TestTheLongestAnswerGrowsWithTheContext(t *testing.T) {
for ctx, want := range map[int]int{2048: 1024, 4096: 1024, 16384: 4096, 32768: 8192, 131072: 8192} {
if got := maxOutput(ctx); got != want {
t.Errorf("context %d: max output %d, want %d", ctx, got, want)
}
}
}
+1 -1
View File
@@ -210,7 +210,7 @@ func (r *Router) Handler() http.Handler {
pages.ServedModels(r.exampleModels)
pages.SetModelFacts(func(name string) web.ModelFacts {
_, m := r.resolve(name)
return web.ModelFacts{ContextLength: m.ContextLength, Tools: m.Capabilities.Tools}
return web.ModelFacts{ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Tools: m.Capabilities.Tools}
})
pages.SetPersonalOff(r.personalOff)
pages.SetLlamaCppVersion(r.llamaVersion)
+15 -2
View File
@@ -49,7 +49,9 @@ var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
// much one request may hold, and whether its tool calls come back as calls.
type ModelFacts struct {
ContextLength int
Tools bool
// MaxOutput is the longest answer the gateway allows; 0 is no limit.
MaxOutput int
Tools bool
}
// continueYAML is a Continue assistant pointing at the gateway: a file of its
@@ -84,7 +86,7 @@ models:
defaultCompletionOptions:
contextLength: %d
maxTokens: %d
`, f.ContextLength, min(4096, max(256, f.ContextLength/4)))
`, f.ContextLength, maxTokensFor(f))
}
if !f.Tools {
b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" +
@@ -277,3 +279,14 @@ func (p *AccountPages) facts(model string) ModelFacts {
}
return p.modelFacts(model)
}
// maxTokensFor is the answer length Continue asks for: a quarter of the
// context, at most 4096, and never over what the gateway allows — asked for
// more, it refuses the request.
func maxTokensFor(f ModelFacts) int {
n := min(4096, max(256, f.ContextLength/4))
if f.MaxOutput > 0 {
n = min(n, f.MaxOutput)
}
return n
}
+8
View File
@@ -135,3 +135,11 @@ func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) {
t.Errorf("a context nobody knows was written down:\n%s", unknown)
}
}
func TestContinueNeverAsksForMoreThanTheGatewayAllows(t *testing.T) {
// Asked for more than max_output, the gateway refuses the request.
y := continueYAML("https://gw.example", "m", "k", ModelFacts{ContextLength: 32768, MaxOutput: 2048, Tools: true})
if !strings.Contains(y, "maxTokens: 2048") {
t.Errorf("not capped at max_output:\n%s", y)
}
}