From df6d243ee7bba7243aaf3e2f8c7caa9fefd9b5b5 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 09:16:34 +0200 Subject: [PATCH] The Continue configuration says what the catalog says of the model It told Continue every model uses tools and nothing of its context: a model whose calls come back as text stalled Continue's agent, and Continue sent prompts far over a CPU replica's 4096 tokens, each refused. tool_use is now written for a model whose catalog entry says tools, and the context length with a fitting maxTokens whenever the catalog knows it. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/router/handlers.go | 4 +++ internal/web/account.go | 5 +++ internal/web/connect.go | 59 +++++++++++++++++++++++++------- internal/web/connect_test.go | 26 ++++++++++++-- internal/web/i18n_public_test.go | 6 ++-- 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/internal/router/handlers.go b/internal/router/handlers.go index bedf77d..5e2813a 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -208,6 +208,10 @@ func (r *Router) Handler() http.Handler { // So the example on the profile page names a model that is actually // running, rather than a placeholder that gets sent as-is. pages.ServedModels(r.exampleModels) + pages.SetModelFacts(func(name string) web.ModelFacts { + _, m := r.resolve(name) + return web.ModelFacts{ContextLength: m.ContextLength, Tools: m.Capabilities.Tools} + }) pages.SetPersonalOff(r.personalOff) pages.SetLlamaCppVersion(r.llamaVersion) pages.SetPersonalDefaults(r.personalDefaults) diff --git a/internal/web/account.go b/internal/web/account.go index 6a9ec4e..d37b128 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -77,6 +77,8 @@ type AccountPages struct { // field in that snippet is real, so the one placeholder left in it was // copied verbatim and sent as the literal string MODEL. served func() []string + // modelFacts is what the catalog says of a model, for an editor. + modelFacts func(model string) ModelFacts // hasMirror is set when /admin/mirror is registered, so the other admin // pages can link to it and only then. hasMirror bool @@ -132,6 +134,9 @@ type AccountPages struct { // example falls back to a placeholder, which is what it always was. func (p *AccountPages) ServedModels(f func() []string) { p.served = f } +// SetModelFacts supplies what an editor's configuration says of a model. +func (p *AccountPages) SetModelFacts(f func(model string) ModelFacts) { p.modelFacts = f } + // exampleModel picks the model the copy-pasteable example should name. // // A real model over a simulated one is not expressible here — this package does diff --git a/internal/web/connect.go b/internal/web/connect.go index fca1531..9dbb593 100644 --- a/internal/web/connect.go +++ b/internal/web/connect.go @@ -45,10 +45,23 @@ type connectField struct{ Label, Value string } // unquoted: the names this gateway serves, and nothing a shell reads twice. var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`) +// ModelFacts is what an editor's configuration needs to know of a model: how +// much one request may hold, and whether its tool calls come back as calls. +type ModelFacts struct { + ContextLength int + Tools bool +} + // continueYAML is a Continue assistant pointing at the gateway: a file of its // own under ~/.continue/assistants, so the person's config.yaml is not touched. -func continueYAML(base, model, key string) string { - return fmt.Sprintf(`# Written by llmgw for Continue: the gateway at %[1]s. +// +// What it says of the model is what the catalog says. Continue assumed a +// context far larger than a CPU replica's, and every request over it was +// refused; and told a model uses tools when its calls come back as text, +// Continue's agent stalls on them rather than acting. +func continueYAML(base, model, key string, f ModelFacts) string { + var b strings.Builder + fmt.Fprintf(&b, `# Written by llmgw for Continue: the gateway at %[1]s. name: llmgw version: 1.0.0 schema: v1 @@ -62,11 +75,25 @@ models: - chat - edit - apply - capabilities: - - tool_use -# For completion as you type, add a small, fast model here with the role -# autocomplete; a large one makes every keystroke wait for it. `, base, model, key) + if f.Tools { + b.WriteString(" capabilities:\n - tool_use\n") + } + if f.ContextLength > 0 { + fmt.Fprintf(&b, ` # What the gateway lets one request hold; Continue trims to fit. + defaultCompletionOptions: + contextLength: %d + maxTokens: %d +`, f.ContextLength, min(4096, max(256, f.ContextLength/4))) + } + if !f.Tools { + b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" + + "# chats and edits with it but its agent cannot act: choose one that does.\n") + } + b.WriteString(`# For completion as you type, add a small, fast model here with the role +# autocomplete; a large one makes every keystroke wait for it. +`) + return b.String() } // codexTOML is the provider and profile Codex needs, and nothing else of its @@ -87,13 +114,13 @@ model = "%[2]s" } // connectFor builds what the page shows for a tool, in English. -func connectFor(tool, base, model, key string) connectView { - return connectIn(english, tool, base, model, key) +func connectFor(tool, base, model, key string, f ModelFacts) connectView { + return connectIn(english, tool, base, model, key, f) } // connectIn builds what the page shows for a tool, its sentences in l. The // files, commands and paths are the same in every language. -func connectIn(l *Lang, tool, base, model, key string) connectView { +func connectIn(l *Lang, tool, base, model, key string, f ModelFacts) connectView { v := connectView{Tool: tool, Model: model, Key: key} for _, t := range editorTools { if t.Key == tool { @@ -104,7 +131,7 @@ func connectIn(l *Lang, tool, base, model, key string) connectView { switch tool { case "continue": v.Files = []connectFile{{Path: "~/.continue/assistants/llmgw.yaml", Name: "llmgw.yaml", - Body: continueYAML(base, model, key)}} + Body: continueYAML(base, model, key, f)}} v.Command = fmt.Sprintf("curl -fsSL '%s' | LLMGW_KEY='%s' sh", script, key) v.Run = l.T("connect.run.continue") case "codex": @@ -173,7 +200,7 @@ func (p *AccountPages) connect(w http.ResponseWriter, r *http.Request) { return } p.log.Info("a user connected an editor", "user", user.ID, "tool", tool, "key", k.ID) - c := connectIn(l, tool, baseURL(r), model, secret) + c := connectIn(l, tool, baseURL(r), model, secret, p.facts(model)) p.render(w, r, sess, user, profileData{Connect: &c}) } @@ -210,7 +237,7 @@ cat > "$file" <<'LLMGW' tmp="$file.tmp" sed "s|__LLMGW_KEY__|$LLMGW_KEY|" "$file" > "$tmp" && mv "$tmp" "$file" echo "Wrote $file. In Continue, choose the llmgw assistant." -`, base, continueYAML(base, model, "__LLMGW_KEY__")) +`, base, continueYAML(base, model, "__LLMGW_KEY__", p.facts(model))) case "codex": body = fmt.Sprintf(`#!/bin/sh # Connects Codex to the llmgw gateway at %[1]s. @@ -242,3 +269,11 @@ echo "Run: LLMGW_API_KEY= codex --profile llmgw" w.Header().Set("Cache-Control", "no-store") _, _ = w.Write([]byte(body)) } + +// facts is what the catalog says of a model, when there is one to ask. +func (p *AccountPages) facts(model string) ModelFacts { + if p.modelFacts == nil { + return ModelFacts{} + } + return p.modelFacts(model) +} diff --git a/internal/web/connect_test.go b/internal/web/connect_test.go index ce2d5e0..be3864c 100644 --- a/internal/web/connect_test.go +++ b/internal/web/connect_test.go @@ -11,7 +11,7 @@ import ( ) func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) { - c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc") + c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc", ModelFacts{}) if !strings.Contains(c.Files[0].Body, "apiBase: https://gw.example/v1") || !strings.Contains(c.Files[0].Body, "apiKey: sk-llmgw-abc") || c.Files[0].Path != "~/.continue/assistants/llmgw.yaml" { t.Errorf("continue = %+v", c.Files) @@ -21,11 +21,11 @@ func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) { if strings.Contains(strings.Split(c.Command, "|")[0], "sk-llmgw-abc") { t.Errorf("the key is in the URL: %s", c.Command) } - x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc") + x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc", ModelFacts{}) if strings.Contains(x.Files[0].Body, "sk-llmgw-abc") || !strings.Contains(x.Files[0].Body, `env_key = "LLMGW_API_KEY"`) { t.Errorf("codex writes the key into its file: %s", x.Files[0].Body) } - cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc") + cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{}) if len(cl.Fields) != 4 || cl.Fields[1].Value != "https://gw.example/v1" || cl.Command != "" { t.Errorf("cline = %+v", cl) } @@ -115,3 +115,23 @@ func TestASetupScriptRefusesANameAShellWouldRead(t *testing.T) { t.Errorf("%d", w.Code) } } + +func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) { + // Told a context far larger than the gateway allows, Continue sends + // prompts that are refused; told a model uses tools when its calls come + // back as text, its agent stalls. + agent := continueYAML("https://gw.example", "coder-30b", "k", ModelFacts{ContextLength: 32768, Tools: true}) + for _, want := range []string{"tool_use", "contextLength: 32768", "maxTokens: 4096"} { + if !strings.Contains(agent, want) { + t.Errorf("%q missing:\n%s", want, agent) + } + } + chat := continueYAML("https://gw.example", "coder", "k", ModelFacts{ContextLength: 4096}) + if strings.Contains(chat, "tool_use") || !strings.Contains(chat, "contextLength: 4096") || + !strings.Contains(chat, "maxTokens: 1024") || !strings.Contains(chat, "its agent cannot act") { + t.Errorf("a model without tools:\n%s", chat) + } + if unknown := continueYAML("https://gw.example", "m", "k", ModelFacts{}); strings.Contains(unknown, "contextLength") { + t.Errorf("a context nobody knows was written down:\n%s", unknown) + } +} diff --git a/internal/web/i18n_public_test.go b/internal/web/i18n_public_test.go index 0b72c83..5b47a58 100644 --- a/internal/web/i18n_public_test.go +++ b/internal/web/i18n_public_test.go @@ -171,18 +171,18 @@ func TestTheProfilePageShowsTurkishNumbersAndDates(t *testing.T) { func TestTheEditorSetupExplainsItselfInTurkish(t *testing.T) { tr := langByCode("tr") - c := connectIn(tr, "cline", "https://gw.example", "m", "sk-llmgw-abc") + c := connectIn(tr, "cline", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{}) if c.Fields[2].Label != "API anahtarı (API Key)" || c.Fields[0].Value != "OpenAI Compatible" || !strings.Contains(c.Run, "dişli simgesine") { t.Errorf("cline in Turkish = %+v", c) } - x := connectIn(tr, "codex", "https://gw.example", "m", "sk-llmgw-abc") + x := connectIn(tr, "codex", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{}) // What is typed or written to a file stays what the tool reads. if x.Run != "LLMGW_API_KEY='sk-llmgw-abc' codex --profile llmgw" || !strings.Contains(x.Files[0].Body, "[profiles.llmgw]") || !strings.Contains(x.Notes[0], "~/.codex/config.toml") { t.Errorf("codex in Turkish = %+v", x) } - if en := connectFor("continue", "https://gw.example", "m", "k"); en.Run != `In Continue, choose the "llmgw" assistant at the top of its panel.` { + if en := connectFor("continue", "https://gw.example", "m", "k", ModelFacts{}); en.Run != `In Continue, choose the "llmgw" assistant at the top of its panel.` { t.Errorf("English Continue: %q", en.Run) } }