Several keys for one outside provider
CI / test (push) Successful in 8m52s

An external model takes api_key_files, each a file and an optional weight,
in place of api_key_file. Each key is an endpoint of its own
(external:MODEL#1, #2), so the router's scoring, retries and breakers work
per key. A key answering 429 is passed over while its breaker is open. A
key answering 401 is passed over too, and external_key_refused is sent,
naming the endpoint and never the key; with a single key the 401 is still
the client's. A key whose file cannot be read is left out, and usage
records which key answered by its endpoint.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-17 08:56:30 +02:00
1 parent 688b205604
commit fe310576e7
14 files changed
+283 -15

No files matched your search

+27
View File
@@ -1533,6 +1533,11 @@ type External struct {
// rather than the key itself, so the key is in neither the catalog, the
// store nor the audit log; empty for a provider that asks for none.
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
// APIKeyFiles are several keys for the provider, in place of
// APIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that
// answers 429 or 401 is passed over while its breaker is open and the
// request goes to another. A provider's limits are per key.
APIKeyFiles []ExternalKey `json:"api_key_files,omitempty" koanf:"api_key_files"`
// Peer names the federated llmgw cluster this model is offered by, for
// a model that federation added (`llmgw ctl federation add`) rather
// than one written by hand. Such a model is named PEER/MODEL, comes and
@@ -1551,6 +1556,14 @@ type External struct {
Output string `json:"output,omitempty" koanf:"output"`
}
// ExternalKey is one of a provider's keys.
type ExternalKey struct {
// File holds the key, on the control plane, as APIKeyFile does.
File string `json:"file" koanf:"file"`
// Weight is its share of the requests against the others'; zero is 1.
Weight float64 `json:"weight,omitempty" koanf:"weight"`
}
// APIComfyUI is External.API for a ComfyUI server.
const APIComfyUI = "comfyui"
@@ -1568,6 +1581,20 @@ func (e *External) validate(model string, profiles int) error {
if strings.TrimSpace(e.Model) == "" {
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
}
if e.APIKeyFile != "" && len(e.APIKeyFiles) > 0 {
return fmt.Errorf("model %q: api_key_file or api_key_files, not both", model)
}
for i, k := range e.APIKeyFiles {
if strings.TrimSpace(k.File) == "" {
return fmt.Errorf("model %q: api_key_files[%d] names no file", model, i)
}
if k.Weight < 0 {
return fmt.Errorf("model %q: api_key_files[%d] has a negative weight", model, i)
}
}
if len(e.APIKeyFiles) > 0 && e.Peer != "" {
return fmt.Errorf("model %q: a federated model is reached with its peer's key, not api_key_files", model)
}
switch e.API {
case "", "openai":
if e.Workflow != "" || e.Output != "" {
+20
View File
@@ -429,3 +429,23 @@ func TestAGuardModelSaysHowItIsReadAndAlwaysRuns(t *testing.T) {
t.Error("a judge is not critical by default")
}
}
func TestAProvidersKeysAreOneListOrOneFile(t *testing.T) {
ext := func(e External) Model {
e.URL, e.Model = "https://api.example.com", "m"
return Model{Name: "cloud", Source: "external", External: &e}
}
for _, bad := range []External{
{APIKeyFile: "/k", APIKeyFiles: []ExternalKey{{File: "/k2"}}},
{APIKeyFiles: []ExternalKey{{File: " "}}},
{APIKeyFiles: []ExternalKey{{File: "/k", Weight: -1}}},
{Peer: "p", APIKeyFiles: []ExternalKey{{File: "/k"}}},
} {
if _, err := New([]Model{ext(bad)}); err == nil {
t.Errorf("taken: %+v", bad)
}
}
if _, err := New([]Model{ext(External{APIKeyFiles: []ExternalKey{{File: "/a", Weight: 2}, {File: "/b"}}})}); err != nil {
t.Error(err)
}
}
+28 -12
View File
@@ -438,15 +438,6 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
continue
}
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := secrets.Read(context.Background(), f)
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
continue
}
key = strings.TrimSpace(string(b))
}
// A peer's endpoint says so in its id, which is what usage records
// as the replica that answered.
id := "external:" + m.Name
@@ -462,14 +453,39 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
}
engine, workflow = endpoint.EngineComfyUI, w
}
out = append(out, endpoint.Info{
base := endpoint.Info{
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Models: append([]string{m.Name}, m.Aliases...),
Engine: engine, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
Workflow: workflow,
})
}
keys := m.External.APIKeyFiles
if len(keys) == 0 {
keys = []catalog.ExternalKey{{File: m.External.APIKeyFile}}
}
for i, k := range keys {
e := base
// One endpoint a key, numbered, so usage says which answered
// and a breaker stands for one key — never the key itself.
if len(m.External.APIKeyFiles) > 0 {
e.ID = fmt.Sprintf("%s#%d", id, i+1)
if k.Weight > 0 {
e.Weight = k.Weight
}
}
if k.File != "" {
b, err := secrets.Read(context.Background(), k.File)
if err != nil {
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name,
"endpoint", e.ID, "err", err)
continue
}
e.APIKey = strings.TrimSpace(string(b))
}
out = append(out, e)
}
}
return out
}
+25
View File
@@ -73,3 +73,28 @@ func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.
}
}
}
// Several keys for one provider: an endpoint each, numbered, with its
// weight; one whose file cannot be read is left out and the others stay.
func TestEachOfAProvidersKeysIsAnEndpointOfItsOwn(t *testing.T) {
dir := t.TempDir()
a, c := filepath.Join(dir, "a.key"), filepath.Join(dir, "c.key")
_ = os.WriteFile(a, []byte("sk-a\n"), 0o600)
_ = os.WriteFile(c, []byte("sk-c"), 0o600)
cat, err := catalog.New([]catalog.Model{{Name: "cloud", Source: "external",
External: &catalog.External{URL: "https://api.example.com", Model: "gpt-4o-mini",
APIKeyFiles: []catalog.ExternalKey{{File: a, Weight: 3}, {File: filepath.Join(dir, "gone")}, {File: c}}}}})
if err != nil {
t.Fatal(err)
}
sink := &recordSink{}
New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler)).Publish(t.Context())
got := map[string]endpoint.Info{}
for _, e := range sink.infos {
got[e.ID] = e
}
if len(got) != 2 || got["external:cloud#1"].APIKey != "sk-a" || got["external:cloud#1"].Weight != 3 ||
got["external:cloud#3"].APIKey != "sk-c" || got["external:cloud#3"].Weight != 1 {
t.Errorf("published %+v", sink.infos)
}
}