The Continue configuration says what the catalog says of the model
CI / test (push) Successful in 2m52s
CI / test (push) Successful in 2m52s
It told Continue every model uses tools and nothing of its context: a model whose calls come back as text stalled Continue's agent, and Continue sent prompts far over a CPU replica's 4096 tokens, each refused. tool_use is now written for a model whose catalog entry says tools, and the context length with a fitting maxTokens whenever the catalog knows it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ef33e52d3d
commit
df6d243ee7
5 files changed
+82
-18
No files matched your search
@@ -208,6 +208,10 @@ func (r *Router) Handler() http.Handler {
|
|||||||
// So the example on the profile page names a model that is actually
|
// So the example on the profile page names a model that is actually
|
||||||
// running, rather than a placeholder that gets sent as-is.
|
// running, rather than a placeholder that gets sent as-is.
|
||||||
pages.ServedModels(r.exampleModels)
|
pages.ServedModels(r.exampleModels)
|
||||||
|
pages.SetModelFacts(func(name string) web.ModelFacts {
|
||||||
|
_, m := r.resolve(name)
|
||||||
|
return web.ModelFacts{ContextLength: m.ContextLength, Tools: m.Capabilities.Tools}
|
||||||
|
})
|
||||||
pages.SetPersonalOff(r.personalOff)
|
pages.SetPersonalOff(r.personalOff)
|
||||||
pages.SetLlamaCppVersion(r.llamaVersion)
|
pages.SetLlamaCppVersion(r.llamaVersion)
|
||||||
pages.SetPersonalDefaults(r.personalDefaults)
|
pages.SetPersonalDefaults(r.personalDefaults)
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ type AccountPages struct {
|
|||||||
// field in that snippet is real, so the one placeholder left in it was
|
// field in that snippet is real, so the one placeholder left in it was
|
||||||
// copied verbatim and sent as the literal string MODEL.
|
// copied verbatim and sent as the literal string MODEL.
|
||||||
served func() []string
|
served func() []string
|
||||||
|
// modelFacts is what the catalog says of a model, for an editor.
|
||||||
|
modelFacts func(model string) ModelFacts
|
||||||
// hasMirror is set when /admin/mirror is registered, so the other admin
|
// hasMirror is set when /admin/mirror is registered, so the other admin
|
||||||
// pages can link to it and only then.
|
// pages can link to it and only then.
|
||||||
hasMirror bool
|
hasMirror bool
|
||||||
@@ -132,6 +134,9 @@ type AccountPages struct {
|
|||||||
// example falls back to a placeholder, which is what it always was.
|
// example falls back to a placeholder, which is what it always was.
|
||||||
func (p *AccountPages) ServedModels(f func() []string) { p.served = f }
|
func (p *AccountPages) ServedModels(f func() []string) { p.served = f }
|
||||||
|
|
||||||
|
// SetModelFacts supplies what an editor's configuration says of a model.
|
||||||
|
func (p *AccountPages) SetModelFacts(f func(model string) ModelFacts) { p.modelFacts = f }
|
||||||
|
|
||||||
// exampleModel picks the model the copy-pasteable example should name.
|
// exampleModel picks the model the copy-pasteable example should name.
|
||||||
//
|
//
|
||||||
// A real model over a simulated one is not expressible here — this package does
|
// A real model over a simulated one is not expressible here — this package does
|
||||||
|
|||||||
+47
-12
@@ -45,10 +45,23 @@ type connectField struct{ Label, Value string }
|
|||||||
// unquoted: the names this gateway serves, and nothing a shell reads twice.
|
// unquoted: the names this gateway serves, and nothing a shell reads twice.
|
||||||
var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
|
var safeModel = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,200}$`)
|
||||||
|
|
||||||
|
// ModelFacts is what an editor's configuration needs to know of a model: how
|
||||||
|
// much one request may hold, and whether its tool calls come back as calls.
|
||||||
|
type ModelFacts struct {
|
||||||
|
ContextLength int
|
||||||
|
Tools bool
|
||||||
|
}
|
||||||
|
|
||||||
// continueYAML is a Continue assistant pointing at the gateway: a file of its
|
// continueYAML is a Continue assistant pointing at the gateway: a file of its
|
||||||
// own under ~/.continue/assistants, so the person's config.yaml is not touched.
|
// own under ~/.continue/assistants, so the person's config.yaml is not touched.
|
||||||
func continueYAML(base, model, key string) string {
|
//
|
||||||
return fmt.Sprintf(`# Written by llmgw for Continue: the gateway at %[1]s.
|
// What it says of the model is what the catalog says. Continue assumed a
|
||||||
|
// context far larger than a CPU replica's, and every request over it was
|
||||||
|
// refused; and told a model uses tools when its calls come back as text,
|
||||||
|
// Continue's agent stalls on them rather than acting.
|
||||||
|
func continueYAML(base, model, key string, f ModelFacts) string {
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, `# Written by llmgw for Continue: the gateway at %[1]s.
|
||||||
name: llmgw
|
name: llmgw
|
||||||
version: 1.0.0
|
version: 1.0.0
|
||||||
schema: v1
|
schema: v1
|
||||||
@@ -62,11 +75,25 @@ models:
|
|||||||
- chat
|
- chat
|
||||||
- edit
|
- edit
|
||||||
- apply
|
- apply
|
||||||
capabilities:
|
|
||||||
- tool_use
|
|
||||||
# For completion as you type, add a small, fast model here with the role
|
|
||||||
# autocomplete; a large one makes every keystroke wait for it.
|
|
||||||
`, base, model, key)
|
`, base, model, key)
|
||||||
|
if f.Tools {
|
||||||
|
b.WriteString(" capabilities:\n - tool_use\n")
|
||||||
|
}
|
||||||
|
if f.ContextLength > 0 {
|
||||||
|
fmt.Fprintf(&b, ` # What the gateway lets one request hold; Continue trims to fit.
|
||||||
|
defaultCompletionOptions:
|
||||||
|
contextLength: %d
|
||||||
|
maxTokens: %d
|
||||||
|
`, f.ContextLength, min(4096, max(256, f.ContextLength/4)))
|
||||||
|
}
|
||||||
|
if !f.Tools {
|
||||||
|
b.WriteString("# This model's catalog entry does not say it makes tool calls, so Continue\n" +
|
||||||
|
"# chats and edits with it but its agent cannot act: choose one that does.\n")
|
||||||
|
}
|
||||||
|
b.WriteString(`# For completion as you type, add a small, fast model here with the role
|
||||||
|
# autocomplete; a large one makes every keystroke wait for it.
|
||||||
|
`)
|
||||||
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// codexTOML is the provider and profile Codex needs, and nothing else of its
|
// codexTOML is the provider and profile Codex needs, and nothing else of its
|
||||||
@@ -87,13 +114,13 @@ model = "%[2]s"
|
|||||||
}
|
}
|
||||||
|
|
||||||
// connectFor builds what the page shows for a tool, in English.
|
// connectFor builds what the page shows for a tool, in English.
|
||||||
func connectFor(tool, base, model, key string) connectView {
|
func connectFor(tool, base, model, key string, f ModelFacts) connectView {
|
||||||
return connectIn(english, tool, base, model, key)
|
return connectIn(english, tool, base, model, key, f)
|
||||||
}
|
}
|
||||||
|
|
||||||
// connectIn builds what the page shows for a tool, its sentences in l. The
|
// connectIn builds what the page shows for a tool, its sentences in l. The
|
||||||
// files, commands and paths are the same in every language.
|
// files, commands and paths are the same in every language.
|
||||||
func connectIn(l *Lang, tool, base, model, key string) connectView {
|
func connectIn(l *Lang, tool, base, model, key string, f ModelFacts) connectView {
|
||||||
v := connectView{Tool: tool, Model: model, Key: key}
|
v := connectView{Tool: tool, Model: model, Key: key}
|
||||||
for _, t := range editorTools {
|
for _, t := range editorTools {
|
||||||
if t.Key == tool {
|
if t.Key == tool {
|
||||||
@@ -104,7 +131,7 @@ func connectIn(l *Lang, tool, base, model, key string) connectView {
|
|||||||
switch tool {
|
switch tool {
|
||||||
case "continue":
|
case "continue":
|
||||||
v.Files = []connectFile{{Path: "~/.continue/assistants/llmgw.yaml", Name: "llmgw.yaml",
|
v.Files = []connectFile{{Path: "~/.continue/assistants/llmgw.yaml", Name: "llmgw.yaml",
|
||||||
Body: continueYAML(base, model, key)}}
|
Body: continueYAML(base, model, key, f)}}
|
||||||
v.Command = fmt.Sprintf("curl -fsSL '%s' | LLMGW_KEY='%s' sh", script, key)
|
v.Command = fmt.Sprintf("curl -fsSL '%s' | LLMGW_KEY='%s' sh", script, key)
|
||||||
v.Run = l.T("connect.run.continue")
|
v.Run = l.T("connect.run.continue")
|
||||||
case "codex":
|
case "codex":
|
||||||
@@ -173,7 +200,7 @@ func (p *AccountPages) connect(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.log.Info("a user connected an editor", "user", user.ID, "tool", tool, "key", k.ID)
|
p.log.Info("a user connected an editor", "user", user.ID, "tool", tool, "key", k.ID)
|
||||||
c := connectIn(l, tool, baseURL(r), model, secret)
|
c := connectIn(l, tool, baseURL(r), model, secret, p.facts(model))
|
||||||
p.render(w, r, sess, user, profileData{Connect: &c})
|
p.render(w, r, sess, user, profileData{Connect: &c})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -210,7 +237,7 @@ cat > "$file" <<'LLMGW'
|
|||||||
tmp="$file.tmp"
|
tmp="$file.tmp"
|
||||||
sed "s|__LLMGW_KEY__|$LLMGW_KEY|" "$file" > "$tmp" && mv "$tmp" "$file"
|
sed "s|__LLMGW_KEY__|$LLMGW_KEY|" "$file" > "$tmp" && mv "$tmp" "$file"
|
||||||
echo "Wrote $file. In Continue, choose the llmgw assistant."
|
echo "Wrote $file. In Continue, choose the llmgw assistant."
|
||||||
`, base, continueYAML(base, model, "__LLMGW_KEY__"))
|
`, base, continueYAML(base, model, "__LLMGW_KEY__", p.facts(model)))
|
||||||
case "codex":
|
case "codex":
|
||||||
body = fmt.Sprintf(`#!/bin/sh
|
body = fmt.Sprintf(`#!/bin/sh
|
||||||
# Connects Codex to the llmgw gateway at %[1]s.
|
# Connects Codex to the llmgw gateway at %[1]s.
|
||||||
@@ -242,3 +269,11 @@ echo "Run: LLMGW_API_KEY=<your key> codex --profile llmgw"
|
|||||||
w.Header().Set("Cache-Control", "no-store")
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
_, _ = w.Write([]byte(body))
|
_, _ = w.Write([]byte(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// facts is what the catalog says of a model, when there is one to ask.
|
||||||
|
func (p *AccountPages) facts(model string) ModelFacts {
|
||||||
|
if p.modelFacts == nil {
|
||||||
|
return ModelFacts{}
|
||||||
|
}
|
||||||
|
return p.modelFacts(model)
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) {
|
func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) {
|
||||||
c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc")
|
c := connectFor("continue", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc", ModelFacts{})
|
||||||
if !strings.Contains(c.Files[0].Body, "apiBase: https://gw.example/v1") || !strings.Contains(c.Files[0].Body, "apiKey: sk-llmgw-abc") ||
|
if !strings.Contains(c.Files[0].Body, "apiBase: https://gw.example/v1") || !strings.Contains(c.Files[0].Body, "apiKey: sk-llmgw-abc") ||
|
||||||
c.Files[0].Path != "~/.continue/assistants/llmgw.yaml" {
|
c.Files[0].Path != "~/.continue/assistants/llmgw.yaml" {
|
||||||
t.Errorf("continue = %+v", c.Files)
|
t.Errorf("continue = %+v", c.Files)
|
||||||
@@ -21,11 +21,11 @@ func TestEachToolGetsItsConfigurationWithTheKey(t *testing.T) {
|
|||||||
if strings.Contains(strings.Split(c.Command, "|")[0], "sk-llmgw-abc") {
|
if strings.Contains(strings.Split(c.Command, "|")[0], "sk-llmgw-abc") {
|
||||||
t.Errorf("the key is in the URL: %s", c.Command)
|
t.Errorf("the key is in the URL: %s", c.Command)
|
||||||
}
|
}
|
||||||
x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc")
|
x := connectFor("codex", "https://gw.example", "qwen2.5-coder-7b", "sk-llmgw-abc", ModelFacts{})
|
||||||
if strings.Contains(x.Files[0].Body, "sk-llmgw-abc") || !strings.Contains(x.Files[0].Body, `env_key = "LLMGW_API_KEY"`) {
|
if strings.Contains(x.Files[0].Body, "sk-llmgw-abc") || !strings.Contains(x.Files[0].Body, `env_key = "LLMGW_API_KEY"`) {
|
||||||
t.Errorf("codex writes the key into its file: %s", x.Files[0].Body)
|
t.Errorf("codex writes the key into its file: %s", x.Files[0].Body)
|
||||||
}
|
}
|
||||||
cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc")
|
cl := connectFor("cline", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{})
|
||||||
if len(cl.Fields) != 4 || cl.Fields[1].Value != "https://gw.example/v1" || cl.Command != "" {
|
if len(cl.Fields) != 4 || cl.Fields[1].Value != "https://gw.example/v1" || cl.Command != "" {
|
||||||
t.Errorf("cline = %+v", cl)
|
t.Errorf("cline = %+v", cl)
|
||||||
}
|
}
|
||||||
@@ -115,3 +115,23 @@ func TestASetupScriptRefusesANameAShellWouldRead(t *testing.T) {
|
|||||||
t.Errorf("%d", w.Code)
|
t.Errorf("%d", w.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestContinueIsToldTheModelsContextAndWhetherItCanUseTools(t *testing.T) {
|
||||||
|
// Told a context far larger than the gateway allows, Continue sends
|
||||||
|
// prompts that are refused; told a model uses tools when its calls come
|
||||||
|
// back as text, its agent stalls.
|
||||||
|
agent := continueYAML("https://gw.example", "coder-30b", "k", ModelFacts{ContextLength: 32768, Tools: true})
|
||||||
|
for _, want := range []string{"tool_use", "contextLength: 32768", "maxTokens: 4096"} {
|
||||||
|
if !strings.Contains(agent, want) {
|
||||||
|
t.Errorf("%q missing:\n%s", want, agent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chat := continueYAML("https://gw.example", "coder", "k", ModelFacts{ContextLength: 4096})
|
||||||
|
if strings.Contains(chat, "tool_use") || !strings.Contains(chat, "contextLength: 4096") ||
|
||||||
|
!strings.Contains(chat, "maxTokens: 1024") || !strings.Contains(chat, "its agent cannot act") {
|
||||||
|
t.Errorf("a model without tools:\n%s", chat)
|
||||||
|
}
|
||||||
|
if unknown := continueYAML("https://gw.example", "m", "k", ModelFacts{}); strings.Contains(unknown, "contextLength") {
|
||||||
|
t.Errorf("a context nobody knows was written down:\n%s", unknown)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -171,18 +171,18 @@ func TestTheProfilePageShowsTurkishNumbersAndDates(t *testing.T) {
|
|||||||
|
|
||||||
func TestTheEditorSetupExplainsItselfInTurkish(t *testing.T) {
|
func TestTheEditorSetupExplainsItselfInTurkish(t *testing.T) {
|
||||||
tr := langByCode("tr")
|
tr := langByCode("tr")
|
||||||
c := connectIn(tr, "cline", "https://gw.example", "m", "sk-llmgw-abc")
|
c := connectIn(tr, "cline", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{})
|
||||||
if c.Fields[2].Label != "API anahtarı (API Key)" || c.Fields[0].Value != "OpenAI Compatible" ||
|
if c.Fields[2].Label != "API anahtarı (API Key)" || c.Fields[0].Value != "OpenAI Compatible" ||
|
||||||
!strings.Contains(c.Run, "dişli simgesine") {
|
!strings.Contains(c.Run, "dişli simgesine") {
|
||||||
t.Errorf("cline in Turkish = %+v", c)
|
t.Errorf("cline in Turkish = %+v", c)
|
||||||
}
|
}
|
||||||
x := connectIn(tr, "codex", "https://gw.example", "m", "sk-llmgw-abc")
|
x := connectIn(tr, "codex", "https://gw.example", "m", "sk-llmgw-abc", ModelFacts{})
|
||||||
// What is typed or written to a file stays what the tool reads.
|
// What is typed or written to a file stays what the tool reads.
|
||||||
if x.Run != "LLMGW_API_KEY='sk-llmgw-abc' codex --profile llmgw" || !strings.Contains(x.Files[0].Body, "[profiles.llmgw]") ||
|
if x.Run != "LLMGW_API_KEY='sk-llmgw-abc' codex --profile llmgw" || !strings.Contains(x.Files[0].Body, "[profiles.llmgw]") ||
|
||||||
!strings.Contains(x.Notes[0], "~/.codex/config.toml") {
|
!strings.Contains(x.Notes[0], "~/.codex/config.toml") {
|
||||||
t.Errorf("codex in Turkish = %+v", x)
|
t.Errorf("codex in Turkish = %+v", x)
|
||||||
}
|
}
|
||||||
if en := connectFor("continue", "https://gw.example", "m", "k"); en.Run != `In Continue, choose the "llmgw" assistant at the top of its panel.` {
|
if en := connectFor("continue", "https://gw.example", "m", "k", ModelFacts{}); en.Run != `In Continue, choose the "llmgw" assistant at the top of its panel.` {
|
||||||
t.Errorf("English Continue: %q", en.Run)
|
t.Errorf("English Continue: %q", en.Run)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user