serve: tool calls declared, slots that hold the context, longer answers

A --context-length 32768 --ctx-pool 32768 model got llama.cpp's default four
slots of 8192, and refused what the gateway admitted; --parallel is now
what the pool holds of the context. max_output was 1024 for every model,
which cut an editor's agent off mid-file; it is now a quarter of the
context, from 1024 to 8192. --tools writes capabilities.tools, which the
Continue configuration now reads, as it reads max_output.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 09:30:40 +02:00
1 parent df6d243ee7
commit 481523ca46
8 files changed
+111 -8

No files matched your search

+15 -2
View File
@@ -49,7 +49,9 @@ var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
// much one request may hold, and whether its tool calls come back as calls.
type ModelFacts struct {
ContextLength int
Tools bool
// MaxOutput is the longest answer the gateway allows; 0 is no limit.
MaxOutput int
Tools bool
}
// continueYAML is a Continue assistant pointing at the gateway: a file of its
@@ -84,7 +86,7 @@ models:
defaultCompletionOptions:
contextLength: %d
maxTokens: %d
`, f.ContextLength, min(4096, max(256, f.ContextLength/4)))
`, f.ContextLength, maxTokensFor(f))
}
if !f.Tools {
b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" +
@@ -277,3 +279,14 @@ func (p *AccountPages) facts(model string) ModelFacts {
}
return p.modelFacts(model)
}
// maxTokensFor is the answer length Continue asks for: a quarter of the
// context, at most 4096, and never over what the gateway allows — asked for
// more, it refuses the request.
func maxTokensFor(f ModelFacts) int {
n := min(4096, max(256, f.ContextLength/4))
if f.MaxOutput > 0 {
n = min(n, f.MaxOutput)
}
return n
}
+8
View File
@@ -135,3 +135,11 @@ func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) {
t.Errorf("a context nobody knows was written down:\n%s", unknown)
}
}
func TestContinueNeverAsksForMoreThanTheGatewayAllows(t *testing.T) {
// Asked for more than max_output, the gateway refuses the request.
y := continueYAML("https://gw.example", "m", "k", ModelFacts{ContextLength: 32768, MaxOutput: 2048, Tools: true})
if !strings.Contains(y, "maxTokens: 2048") {
t.Errorf("not capped at max_output:\n%s", y)
}
}