Several keys for one outside provider
An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
@@ -6,11 +6,15 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
@@ -87,3 +91,68 @@ func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two keys for one provider, one of them at its limit, or refused: every
|
||||
// request is answered by the other key, the failing one's breaker opens, and
|
||||
// a refused key is said, by its endpoint and never its value.
|
||||
func TestAKeyAtItsLimitOrRefusedIsPassedOverForAnother(t *testing.T) {
|
||||
for _, status := range []int{http.StatusTooManyRequests, http.StatusUnauthorized} {
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") == "Bearer key-a" {
|
||||
http.Error(w, "no", status)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"model":"m","choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
|
||||
}))
|
||||
t.Cleanup(provider.Close)
|
||||
cfg := baseConfig()
|
||||
cfg.Models = []catalog.Model{{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: provider.URL, Model: "m"}}}
|
||||
rt := newTestRouter(t, cfg)
|
||||
key := func(id, k string) endpoint.Info {
|
||||
return endpoint.Info{ID: id, URL: provider.URL, Models: []string{"cloud"}, APIKey: k,
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "m"}
|
||||
}
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{key("external:cloud#1", "key-a"), key("external:cloud#2", "key-b")})
|
||||
n, events := notifytest.Catch(t)
|
||||
rt.SetNotifier(n)
|
||||
|
||||
// Until the failing key's breaker opens: how many requests the
|
||||
// scheduler sends it before then is its own choice.
|
||||
open := func() bool {
|
||||
for _, s := range rt.reg.Snapshots("cloud") {
|
||||
if s.ID == "external:cloud#1" && s.Breaker != breaker.Closed {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for i := 0; i < 100 && !open(); i++ {
|
||||
if rec := post(t, rt.Handler(), map[string]any{"model": "cloud", "messages": []any{}}, nil); rec.Code != http.StatusOK {
|
||||
t.Fatalf("key-a answering %d: request %d got %d: %s", status, i, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
for _, s := range rt.reg.Snapshots("cloud") {
|
||||
if s.ID == "external:cloud#1" && s.Breaker == breaker.Closed {
|
||||
t.Errorf("%d: the failing key's breaker is still closed", status)
|
||||
}
|
||||
if s.ID == "external:cloud#2" && s.Breaker != breaker.Closed {
|
||||
t.Errorf("%d: the good key's breaker opened", status)
|
||||
}
|
||||
}
|
||||
var refused []notify.Event
|
||||
for _, e := range events() {
|
||||
if e.Kind == notify.ExternalKeyRefused {
|
||||
refused = append(refused, e)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case status == http.StatusTooManyRequests && len(refused) != 0:
|
||||
t.Errorf("a key at its limit was said to be refused: %+v", refused)
|
||||
case status == http.StatusUnauthorized && (len(refused) != 1 || refused[0].Subject != "external:cloud#1" ||
|
||||
strings.Contains(refused[0].Message, "key-a")):
|
||||
t.Errorf("a refused key was said as %+v", refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user