serve: tool calls declared, slots that hold the context, longer answers
CI / test (push) Failing after 2m41s
CI / test (push) Failing after 2m41s
A --context-length 32768 --ctx-pool 32768 model got llama.cpp's default four slots of 8192, and refused what the gateway admitted; --parallel is now what the pool holds of the context. max_output was 1024 for every model, which cut an editor's agent off mid-file; it is now a quarter of the context, from 1024 to 8192. --tools writes capabilities.tools, which the Continue configuration now reads, as it reads max_output. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
df6d243ee7
commit
481523ca46
8 files changed
+111
-8
No files matched your search
@@ -53,8 +53,22 @@ Two control planes, run in anger on the reference cluster.
|
||||
dashboard as well as the API; a proxy left pointing at a standby showed an
|
||||
empty cluster. The leader trusts the control planes' `X-Forwarded-For`.
|
||||
|
||||
- `ctl model serve --tools` says in the catalog that a model makes tool
|
||||
calls, for agents and editors.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Continue's agent did nothing.** The configuration the portal writes told
|
||||
Continue every model uses tools, and nothing of its context: a model whose
|
||||
calls came back as text stalled the agent, and prompts over a CPU
|
||||
replica's 4096 tokens were refused. It now says `tool_use` only for a
|
||||
model whose catalog says tools, and the context and longest answer the
|
||||
gateway allows.
|
||||
- `ctl model serve` gave llama.cpp four slots whatever was asked: a
|
||||
`--context-length 32768 --ctx-pool 32768` model had slots of 8192 and
|
||||
refused what the gateway admitted. Slots are now what the pool holds of
|
||||
the context. The longest answer grows with the context (a quarter, from
|
||||
1024 to 8192) rather than being 1024 for every model.
|
||||
- `ctl upgrade` upgrades the leader last wherever it is run; run on the
|
||||
standby, it upgraded the leader first and the leadership moved twice.
|
||||
- **Failover took 40 seconds; it takes a few.** A stopping control plane held
|
||||
|
||||
@@ -184,6 +184,15 @@ asks for, send the results back. Whether a model calls tools well is the
|
||||
model's business, not the gateway's; ask your operator which of the served
|
||||
models are set up for it.
|
||||
|
||||
**Editors' agents need such a model.** Continue's Agent mode, Cline and
|
||||
Codex act only through tool calls: with a model whose calls come back as
|
||||
text — `<tool_call>{...}` or a JSON block in the answer, `tool_calls` empty —
|
||||
they chat but never read a file or run a command. Qwen2.5-Coder is one;
|
||||
Qwen3-Coder is not. The configuration the portal writes for Continue says
|
||||
`tool_use` only for a model whose catalog entry says tools, and gives
|
||||
Continue the model's context and longest answer, so it never sends a request
|
||||
the gateway refuses.
|
||||
|
||||
```python
|
||||
import json
|
||||
|
||||
|
||||
@@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||
f.BoolVar(&req.Tools, "tools", false,
|
||||
"say it makes tool calls (for agents and editors); check a running one with llmgw ctl tool-check")
|
||||
f.StringVar(&req.GuardFormat, "guard-format", "",
|
||||
"serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients")
|
||||
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||
|
||||
@@ -140,9 +140,14 @@ type Request struct {
|
||||
// llamaguard3, shieldgemma or json. Empty serves clients.
|
||||
GuardFormat string `json:"guard_format,omitempty"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide;
|
||||
// llama.cpp is given as many slots as the pool holds requests.
|
||||
CtxPool int `json:"ctx_pool,omitempty"`
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
// Tools says the model makes tool calls the engine returns as calls: it
|
||||
// is written into the catalog, where clients and editors read it. Check
|
||||
// a running replica with llmgw ctl tool-check before saying so.
|
||||
Tools bool `json:"tools,omitempty"`
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
@@ -261,6 +266,10 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
if req.ContextLength <= 0 {
|
||||
req.ContextLength = 4096
|
||||
}
|
||||
if req.ContextLength > req.CtxPool {
|
||||
return Job{}, fmt.Errorf("a request of %d tokens cannot fit a pool of %d: give --ctx-pool at least --context-length",
|
||||
req.ContextLength, req.CtxPool)
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.seq++
|
||||
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
|
||||
@@ -545,12 +554,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
|
||||
switch {
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
p.Args = llamaArgs(req)
|
||||
p.MinMemoryGB, p.CPUCores = memGB, cores
|
||||
case o.Engine == "llamacpp":
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
|
||||
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
p.Args = llamaArgs(req)
|
||||
default:
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
|
||||
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
|
||||
@@ -564,6 +573,23 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
|
||||
return out
|
||||
}
|
||||
|
||||
// llamaArgs size llama.cpp's pool and its slots together. The pool is shared
|
||||
// by the slots, so slots of the context the catalog promises are as many as
|
||||
// the pool holds: the engine's default of four turned a 32768 pool into
|
||||
// slots of 8192, and the gateway, told 32768, sent requests the engine then
|
||||
// refused.
|
||||
func llamaArgs(req Request) []string {
|
||||
return []string{"--ctx-size", strconv.Itoa(req.CtxPool),
|
||||
"--parallel", strconv.Itoa(max(1, req.CtxPool/req.ContextLength))}
|
||||
}
|
||||
|
||||
// maxOutput is the longest answer the catalog allows: a quarter of the
|
||||
// context, from 1024 to 8192. 1024 everywhere cut an editor's agent off in
|
||||
// the middle of the file it was writing.
|
||||
func maxOutput(contextLength int) int {
|
||||
return min(8192, max(1024, contextLength/4))
|
||||
}
|
||||
|
||||
// addAndStart puts the model in the catalog with its profile, waits for a
|
||||
// replica and asks it to answer.
|
||||
// hardwareExists is whether the cluster has what the ways chosen need, busy
|
||||
@@ -583,10 +609,10 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: maxOutput(req.ContextLength),
|
||||
Profiles: profs,
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
}
|
||||
if req.GuardFormat != "" {
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLlamaCppIsGivenTheSlotsTheCatalogPromises(t *testing.T) {
|
||||
// A 32768 pool for 32768-token requests is one slot: the engine's own
|
||||
// four made slots of 8192, and requests the gateway admitted were refused.
|
||||
for _, c := range []struct {
|
||||
pool, ctx int
|
||||
want []string
|
||||
}{
|
||||
{32768, 32768, []string{"--ctx-size", "32768", "--parallel", "1"}},
|
||||
{16384, 4096, []string{"--ctx-size", "16384", "--parallel", "4"}},
|
||||
{65536, 16384, []string{"--ctx-size", "65536", "--parallel", "4"}},
|
||||
} {
|
||||
if got := llamaArgs(Request{CtxPool: c.pool, ContextLength: c.ctx}); !slices.Equal(got, c.want) {
|
||||
t.Errorf("pool %d, context %d: %v", c.pool, c.ctx, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLongestAnswerGrowsWithTheContext(t *testing.T) {
|
||||
for ctx, want := range map[int]int{2048: 1024, 4096: 1024, 16384: 4096, 32768: 8192, 131072: 8192} {
|
||||
if got := maxOutput(ctx); got != want {
|
||||
t.Errorf("context %d: max output %d, want %d", ctx, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -210,7 +210,7 @@ func (r *Router) Handler() http.Handler {
|
||||
pages.ServedModels(r.exampleModels)
|
||||
pages.SetModelFacts(func(name string) web.ModelFacts {
|
||||
_, m := r.resolve(name)
|
||||
return web.ModelFacts{ContextLength: m.ContextLength, Tools: m.Capabilities.Tools}
|
||||
return web.ModelFacts{ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Tools: m.Capabilities.Tools}
|
||||
})
|
||||
pages.SetPersonalOff(r.personalOff)
|
||||
pages.SetLlamaCppVersion(r.llamaVersion)
|
||||
|
||||
+15
-2
@@ -49,7 +49,9 @@ var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
|
||||
// much one request may hold, and whether its tool calls come back as calls.
|
||||
type ModelFacts struct {
|
||||
ContextLength int
|
||||
Tools bool
|
||||
// MaxOutput is the longest answer the gateway allows; 0 is no limit.
|
||||
MaxOutput int
|
||||
Tools bool
|
||||
}
|
||||
|
||||
// continueYAML is a Continue assistant pointing at the gateway: a file of its
|
||||
@@ -84,7 +86,7 @@ models:
|
||||
defaultCompletionOptions:
|
||||
contextLength: %d
|
||||
maxTokens: %d
|
||||
`, f.ContextLength, min(4096, max(256, f.ContextLength/4)))
|
||||
`, f.ContextLength, maxTokensFor(f))
|
||||
}
|
||||
if !f.Tools {
|
||||
b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" +
|
||||
@@ -277,3 +279,14 @@ func (p *AccountPages) facts(model string) ModelFacts {
|
||||
}
|
||||
return p.modelFacts(model)
|
||||
}
|
||||
|
||||
// maxTokensFor is the answer length Continue asks for: a quarter of the
|
||||
// context, at most 4096, and never over what the gateway allows — asked for
|
||||
// more, it refuses the request.
|
||||
func maxTokensFor(f ModelFacts) int {
|
||||
n := min(4096, max(256, f.ContextLength/4))
|
||||
if f.MaxOutput > 0 {
|
||||
n = min(n, f.MaxOutput)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -135,3 +135,11 @@ func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) {
|
||||
t.Errorf("a context nobody knows was written down:\n%s", unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinueNeverAsksForMoreThanTheGatewayAllows(t *testing.T) {
|
||||
// Asked for more than max_output, the gateway refuses the request.
|
||||
y := continueYAML("https://gw.example", "m", "k", ModelFacts{ContextLength: 32768, MaxOutput: 2048, Tools: true})
|
||||
if !strings.Contains(y, "maxTokens: 2048") {
|
||||
t.Errorf("not capped at max_output:\n%s", y)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user