Several keys for one outside provider

An external model takes api_key_files, each a file and an optional weight,
in place of api_key_file. Each key is an endpoint of its own
(external:MODEL#1, #2), so the router's scoring, retries and breakers work
per key. A key answering 429 is passed over while its breaker is open. A
key answering 401 is passed over too, and external_key_refused is sent,
naming the endpoint and never the key; with a single key the 401 is still
the client's. A key whose file cannot be read is left out, and usage
records which key answered by its endpoint.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-17 08:56:30 +02:00
1 parent 688b205604
commit fe310576e7
14 files changed
+283 -15

No files matched your search

+11
View File
@@ -131,7 +131,18 @@ const defaultActivationWait = 45 * time.Second
// SetNotifier tells the accountant where to say a budget is spent. Nothing
// without a store, where there are no budgets.
func (r *Router) SetNotifier(n *notify.Notifier) {
r.notifier = n
if r.acct != nil {
r.acct.SetNotifier(n)
}
}
// keyRefused says that an outside provider answered 401 to one of a model's
// keys, by its endpoint, which numbers it: never the key.
func (r *Router) keyRefused(model, endpointID string) {
r.log.Warn("an outside provider refused a key", "model", model, "endpoint", endpointID)
r.notifier.Notify(notify.ExternalKeyRefused, endpointID,
"the provider of "+model+" answered 401 to the key of "+endpointID+
": revoked, expired or mistyped; replace its file on the control plane",
map[string]any{"model": model, "endpoint": endpointID})
}
+69
View File
@@ -6,11 +6,15 @@ import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
@@ -87,3 +91,68 @@ func TestAnExternalModelIsNotAModelWithProfiles(t *testing.T) {
}
}
}
// Two keys for one provider, one of them at its limit, or refused: every
// request is answered by the other key, the failing one's breaker opens, and
// a refused key is said, by its endpoint and never its value.
func TestAKeyAtItsLimitOrRefusedIsPassedOverForAnother(t *testing.T) {
for _, status := range []int{http.StatusTooManyRequests, http.StatusUnauthorized} {
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") == "Bearer key-a" {
http.Error(w, "no", status)
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"model":"m","choices":[{"message":{"content":"ok"}}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
}))
t.Cleanup(provider.Close)
cfg := baseConfig()
cfg.Models = []catalog.Model{{Name: "cloud", Source: "external",
External: &catalog.External{URL: provider.URL, Model: "m"}}}
rt := newTestRouter(t, cfg)
key := func(id, k string) endpoint.Info {
return endpoint.Info{ID: id, URL: provider.URL, Models: []string{"cloud"}, APIKey: k,
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1, UpstreamModel: "m"}
}
rt.SetEndpoints(t.Context(), []endpoint.Info{key("external:cloud#1", "key-a"), key("external:cloud#2", "key-b")})
n, events := notifytest.Catch(t)
rt.SetNotifier(n)
// Until the failing key's breaker opens: how many requests the
// scheduler sends it before then is its own choice.
open := func() bool {
for _, s := range rt.reg.Snapshots("cloud") {
if s.ID == "external:cloud#1" && s.Breaker != breaker.Closed {
return true
}
}
return false
}
for i := 0; i < 100 && !open(); i++ {
if rec := post(t, rt.Handler(), map[string]any{"model": "cloud", "messages": []any{}}, nil); rec.Code != http.StatusOK {
t.Fatalf("key-a answering %d: request %d got %d: %s", status, i, rec.Code, rec.Body.String())
}
}
for _, s := range rt.reg.Snapshots("cloud") {
if s.ID == "external:cloud#1" && s.Breaker == breaker.Closed {
t.Errorf("%d: the failing key's breaker is still closed", status)
}
if s.ID == "external:cloud#2" && s.Breaker != breaker.Closed {
t.Errorf("%d: the good key's breaker opened", status)
}
}
var refused []notify.Event
for _, e := range events() {
if e.Kind == notify.ExternalKeyRefused {
refused = append(refused, e)
}
}
switch {
case status == http.StatusTooManyRequests && len(refused) != 0:
t.Errorf("a key at its limit was said to be refused: %+v", refused)
case status == http.StatusUnauthorized && (len(refused) != 1 || refused[0].Subject != "external:cloud#1" ||
strings.Contains(refused[0].Message, "key-a")):
t.Errorf("a refused key was said as %+v", refused)
}
}
}
+12
View File
@@ -34,6 +34,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
@@ -116,6 +117,8 @@ type Router struct {
// federation is which cluster this is, to its peers; nil until said.
federation atomic.Pointer[federationID]
acct *accounting.Accountant
// notifier says what the operator must act on; nil says nothing.
notifier *notify.Notifier
// guard judges requests and answers; nil judges nothing. guardLibrary
// is the policies it was built from, for the pages that show them.
guard guard.Guard
@@ -1193,6 +1196,12 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
// restart. Not relayed; loaded again and tried again.
preq.RetryStatus = http.StatusNotFound
}
// One of an outside provider's several keys refused: the request
// is the provider's to answer with another of them, and the key's
// breaker counts it. With one key the 401 is the client's to see.
if dec.Endpoint.Engine == endpoint.EngineExternal && strings.Contains(dec.Endpoint.ID, "#") {
preq.RetryStatus = http.StatusUnauthorized
}
if dec.Endpoint.Peer != "" {
preq.Extra = r.peerHeaders(ctx, dec.Endpoint, req.Header)
}
@@ -1209,6 +1218,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
} else {
out, perr = r.px.Do(uctx, w, preq)
}
if dec.Endpoint.Engine == endpoint.EngineExternal && out.Status == http.StatusUnauthorized {
r.keyRefused(model, dec.Endpoint.ID)
}
releaseAdapter()
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
endSpan(uspan, perr)