Several keys for one outside provider
An external model takes api_key_files, each a file and an optional weight, in place of api_key_file. Each key is an endpoint of its own (external:MODEL#1, #2), so the router's scoring, retries and breakers work per key. A key answering 429 is passed over while its breaker is open. A key answering 401 is passed over too, and external_key_refused is sent, naming the endpoint and never the key; with a single key the 401 is still the client's. A key whose file cannot be read is left out, and usage records which key answered by its endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
688b205604
commit
fe310576e7
14 files changed
+283
-15
No files matched your search
@@ -73,3 +73,28 @@ func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Several keys for one provider: an endpoint each, numbered, with its
|
||||
// weight; one whose file cannot be read is left out and the others stay.
|
||||
func TestEachOfAProvidersKeysIsAnEndpointOfItsOwn(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a, c := filepath.Join(dir, "a.key"), filepath.Join(dir, "c.key")
|
||||
_ = os.WriteFile(a, []byte("sk-a\n"), 0o600)
|
||||
_ = os.WriteFile(c, []byte("sk-c"), 0o600)
|
||||
cat, err := catalog.New([]catalog.Model{{Name: "cloud", Source: "external",
|
||||
External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini",
|
||||
APIKeyFiles: []catalog.ExternalKey{{File: a, Weight: 3}, {File: filepath.Join(dir, "gone")}, {File: c}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &recordSink{}
|
||||
New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler)).Publish(t.Context())
|
||||
got := map[string]endpoint.Info{}
|
||||
for _, e := range sink.infos {
|
||||
got[e.ID] = e
|
||||
}
|
||||
if len(got) != 2 || got["external:cloud#1"].APIKey != "sk-a" || got["external:cloud#1"].Weight != 3 ||
|
||||
got["external:cloud#3"].APIKey != "sk-c" || got["external:cloud#3"].Weight != 1 {
|
||||
t.Errorf("published %+v", sink.infos)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user