diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ea4fee..f658817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,8 +53,22 @@ Two control planes, run in anger on the reference cluster. dashboard as well as the API; a proxy left pointing at a standby showed an empty cluster. The leader trusts the control planes' `X-Forwarded-For`. +- `ctl model serve --tools` says in the catalog that a model makes tool + calls, for agents and editors. + ### Fixed +- **Continue's agent did nothing.** The configuration the portal writes told + Continue every model uses tools, and nothing of its context: a model whose + calls came back as text stalled the agent, and prompts over a CPU + replica's 4096 tokens were refused. It now says `tool_use` only for a + model whose catalog says tools, and the context and longest answer the + gateway allows. +- `ctl model serve` gave llama.cpp four slots whatever was asked: a + `--context-length 32768 --ctx-pool 32768` model had slots of 8192 and + refused what the gateway admitted. Slots are now what the pool holds of + the context. The longest answer grows with the context (a quarter, from + 1024 to 8192) rather than being 1024 for every model. - `ctl upgrade` upgrades the leader last wherever it is run; run on the standby, it upgraded the leader first and the leadership moved twice. - **Failover took 40 seconds; it takes a few.** A stopping control plane held diff --git a/docs/clients.md b/docs/clients.md index 59e3dc5..f1247a7 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -184,6 +184,15 @@ asks for, send the results back. Whether a model calls tools well is the model's business, not the gateway's; ask your operator which of the served models are set up for it. +**Editors' agents need such a model.** Continue's Agent mode, Cline and +Codex act only through tool calls: with a model whose calls come back as +text — `{...}` or a JSON block in the answer, `tool_calls` empty — +they chat but never read a file or run a command. Qwen2.5-Coder is one; +Qwen3-Coder is not. The configuration the portal writes for Continue says +`tool_use` only for a model whose catalog entry says tools, and gives +Continue the model's context and longest answer, so it never sends a request +the gateway refuses. + ```python import json diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 3ccaa7f..47fdb30 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -73,6 +73,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run") + f.BoolVar(&req.Tools, "tools", false, + "say it makes tool calls (for agents and editors); check a running one with llmgw ctl tool-check") f.StringVar(&req.GuardFormat, "guard-format", "", "serve it as the guardrails' judge (role: guard), read as llamaguard3, shieldgemma or json; hidden from clients") f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for") diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 42b7eab..fe0a05d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -140,9 +140,14 @@ type Request struct { // llamaguard3, shieldgemma or json. Empty serves clients. GuardFormat string `json:"guard_format,omitempty"` // CtxPool is llama.cpp's --ctx-size; ContextLength what one request may - // use. They default to 16384 and 4096, what four parallel slots divide. + // use. They default to 16384 and 4096, what four parallel slots divide; + // llama.cpp is given as many slots as the pool holds requests. CtxPool int `json:"ctx_pool,omitempty"` ContextLength int `json:"context_length,omitempty"` + // Tools says the model makes tool calls the engine returns as calls: it + // is written into the catalog, where clients and editors read it. Check + // a running replica with llmgw ctl tool-check before saying so. + Tools bool `json:"tools,omitempty"` // CPUCores is what a replica computes on; zero takes half the smallest // node it fits on. CPUCores int `json:"cpu_cores,omitempty"` @@ -261,6 +266,10 @@ func (s *Server) Start(req Request) (Job, error) { if req.ContextLength <= 0 { req.ContextLength = 4096 } + if req.ContextLength > req.CtxPool { + return Job{}, fmt.Errorf("a request of %d tokens cannot fit a pool of %d: give --ctx-pool at least --context-length", + req.ContextLength, req.CtxPool) + } s.mu.Lock() s.seq++ j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req, @@ -545,12 +554,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat switch { case o.CPU: p.Name = "cpu" - p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + p.Args = llamaArgs(req) p.MinMemoryGB, p.CPUCores = memGB, cores case o.Engine == "llamacpp": p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class) p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode - p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)} + p.Args = llamaArgs(req) default: p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class) p.GPUClass, p.TP = o.Class, o.GPUsPerNode @@ -564,6 +573,23 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat return out } +// llamaArgs size llama.cpp's pool and its slots together. The pool is shared +// by the slots, so slots of the context the catalog promises are as many as +// the pool holds: the engine's default of four turned a 32768 pool into +// slots of 8192, and the gateway, told 32768, sent requests the engine then +// refused. +func llamaArgs(req Request) []string { + return []string{"--ctx-size", strconv.Itoa(req.CtxPool), + "--parallel", strconv.Itoa(max(1, req.CtxPool/req.ContextLength))} +} + +// maxOutput is the longest answer the catalog allows: a quarter of the +// context, from 1024 to 8192. 1024 everywhere cut an editor's agent off in +// the middle of the file it was writing. +func maxOutput(contextLength int) int { + return min(8192, max(1024, contextLength/4)) +} + // addAndStart puts the model in the catalog with its profile, waits for a // replica and asks it to answer. // hardwareExists is whether the cluster has what the ways chosen need, busy @@ -583,10 +609,10 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F // 4. The catalog. m := catalog.Model{ Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, - RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, + RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: maxOutput(req.ContextLength), Profiles: profs, Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, - Capabilities: catalog.Capabilities{Vision: req.Projector != ""}, + Capabilities: catalog.Capabilities{Vision: req.Projector != "", Tools: req.Tools}, Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)}, } if req.GuardFormat != "" { diff --git a/internal/control/modelserve/slots_test.go b/internal/control/modelserve/slots_test.go new file mode 100644 index 0000000..f090e58 --- /dev/null +++ b/internal/control/modelserve/slots_test.go @@ -0,0 +1,31 @@ +package modelserve + +import ( + "slices" + "testing" +) + +func TestLlamaCppIsGivenTheSlotsTheCatalogPromises(t *testing.T) { + // A 32768 pool for 32768-token requests is one slot: the engine's own + // four made slots of 8192, and requests the gateway admitted were refused. + for _, c := range []struct { + pool, ctx int + want []string + }{ + {32768, 32768, []string{"--ctx-size", "32768", "--parallel", "1"}}, + {16384, 4096, []string{"--ctx-size", "16384", "--parallel", "4"}}, + {65536, 16384, []string{"--ctx-size", "65536", "--parallel", "4"}}, + } { + if got := llamaArgs(Request{CtxPool: c.pool, ContextLength: c.ctx}); !slices.Equal(got, c.want) { + t.Errorf("pool %d, context %d: %v", c.pool, c.ctx, got) + } + } +} + +func TestTheLongestAnswerGrowsWithTheContext(t *testing.T) { + for ctx, want := range map[int]int{2048: 1024, 4096: 1024, 16384: 4096, 32768: 8192, 131072: 8192} { + if got := maxOutput(ctx); got != want { + t.Errorf("context %d: max output %d, want %d", ctx, got, want) + } + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 5e2813a..4cf1d14 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -210,7 +210,7 @@ func (r *Router) Handler() http.Handler { pages.ServedModels(r.exampleModels) pages.SetModelFacts(func(name string) web.ModelFacts { _, m := r.resolve(name) - return web.ModelFacts{ContextLength: m.ContextLength, Tools: m.Capabilities.Tools} + return web.ModelFacts{ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Tools: m.Capabilities.Tools} }) pages.SetPersonalOff(r.personalOff) pages.SetLlamaCppVersion(r.llamaVersion) diff --git a/internal/web/connect.go b/internal/web/connect.go index 9dbb593..83851ed 100644 --- a/internal/web/connect.go +++ b/internal/web/connect.go @@ -49,7 +49,9 @@ var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`) // much one request may hold, and whether its tool calls come back as calls. type ModelFacts struct { ContextLength int - Tools bool + // MaxOutput is the longest answer the gateway allows; 0 is no limit. + MaxOutput int + Tools bool } // continueYAML is a Continue assistant pointing at the gateway: a file of its @@ -84,7 +86,7 @@ models: defaultCompletionOptions: contextLength: %d maxTokens: %d -`, f.ContextLength, min(4096, max(256, f.ContextLength/4))) +`, f.ContextLength, maxTokensFor(f)) } if !f.Tools { b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" + @@ -277,3 +279,14 @@ func (p *AccountPages) facts(model string) ModelFacts { } return p.modelFacts(model) } + +// maxTokensFor is the answer length Continue asks for: a quarter of the +// context, at most 4096, and never over what the gateway allows — asked for +// more, it refuses the request. +func maxTokensFor(f ModelFacts) int { + n := min(4096, max(256, f.ContextLength/4)) + if f.MaxOutput > 0 { + n = min(n, f.MaxOutput) + } + return n +} diff --git a/internal/web/connect_test.go b/internal/web/connect_test.go index be3864c..8c58342 100644 --- a/internal/web/connect_test.go +++ b/internal/web/connect_test.go @@ -135,3 +135,11 @@ func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) { t.Errorf("a context nobody knows was written down:\n%s", unknown) } } + +func TestContinueNeverAsksForMoreThanTheGatewayAllows(t *testing.T) { + // Asked for more than max_output, the gateway refuses the request. + y := continueYAML("https://gw.example", "m", "k", ModelFacts{ContextLength: 32768, MaxOutput: 2048, Tools: true}) + if !strings.Contains(y, "maxTokens: 2048") { + t.Errorf("not capped at max_output:\n%s", y) + } +}