serve: tool calls declared, slots that hold the context, longer answers
A --context-length 32768 --ctx-pool 32768 model got llama.cpp's default four slots of 8192, and refused what the gateway admitted; --parallel is now what the pool holds of the context. max_output was 1024 for every model, which cut an editor's agent off mid-file; it is now a quarter of the context, from 1024 to 8192. --tools writes capabilities.tools, which the Continue configuration now reads, as it reads max_output. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
df6d243ee7
commit
481523ca46
8 files changed
+111
-8
No files matched your search
+15
-2
@@ -49,7 +49,9 @@ var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
|
||||
// much one request may hold, and whether its tool calls come back as calls.
|
||||
type ModelFacts struct {
|
||||
ContextLength int
|
||||
Tools bool
|
||||
// MaxOutput is the longest answer the gateway allows; 0 is no limit.
|
||||
MaxOutput int
|
||||
Tools bool
|
||||
}
|
||||
|
||||
// continueYAML is a Continue assistant pointing at the gateway: a file of its
|
||||
@@ -84,7 +86,7 @@ models:
|
||||
defaultCompletionOptions:
|
||||
contextLength: %d
|
||||
maxTokens: %d
|
||||
`, f.ContextLength, min(4096, max(256, f.ContextLength/4)))
|
||||
`, f.ContextLength, maxTokensFor(f))
|
||||
}
|
||||
if !f.Tools {
|
||||
b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" +
|
||||
@@ -277,3 +279,14 @@ func (p *AccountPages) facts(model string) ModelFacts {
|
||||
}
|
||||
return p.modelFacts(model)
|
||||
}
|
||||
|
||||
// maxTokensFor is the answer length Continue asks for: a quarter of the
|
||||
// context, at most 4096, and never over what the gateway allows — asked for
|
||||
// more, it refuses the request.
|
||||
func maxTokensFor(f ModelFacts) int {
|
||||
n := min(4096, max(256, f.ContextLength/4))
|
||||
if f.MaxOutput > 0 {
|
||||
n = min(n, f.MaxOutput)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -135,3 +135,11 @@ func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) {
|
||||
t.Errorf("a context nobody knows was written down:\n%s", unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinueNeverAsksForMoreThanTheGatewayAllows(t *testing.T) {
|
||||
// Asked for more than max_output, the gateway refuses the request.
|
||||
y := continueYAML("https://gw.example", "m", "k", ModelFacts{ContextLength: 32768, MaxOutput: 2048, Tools: true})
|
||||
if !strings.Contains(y, "maxTokens: 2048") {
|
||||
t.Errorf("not capped at max_output:\n%s", y)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user